@datafog/fogclaw 0.2.0 → 0.3.0

package/scripts/ci/he-spikes-lint.sh

@@ -1,249 +0,0 @@
-#!/bin/bash
-set -euo pipefail
-
-# ---------------------------------------------------------------------------
-# he-spikes-lint.sh — Lint spike documents under docs/spikes
-# ---------------------------------------------------------------------------
-
-SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
-REPO_ROOT="$(cd "$SCRIPT_DIR/../.." && pwd)"
-
-DEFAULT_CONFIG_PATH="scripts/ci/he-docs-config.json"
-
-# Default required headings (one per line for easy iteration)
-DEFAULT_REQUIRED_HEADINGS=(
-  "## Context"
-  "## Validation Goal"
-  "## Approach"
-  "## Findings"
-  "## Decisions"
-  "## Recommendation"
-  "## Impact on Upstream Docs"
-  "## Spike Code"
-  "## Remaining Unknowns"
-  "## Time Spent"
-  "## Revision Notes"
-)
-
-# Counters
-errors=0
-warnings=0
-
-# ---------------------------------------------------------------------------
-# Helpers
-# ---------------------------------------------------------------------------
-
-gh_annotate() {
-  local level="$1" file="$2" title="$3" msg="$4"
-  if [[ -n "$file" ]]; then
-    echo "::${level} file=${file},title=${title}::${msg}"
-  else
-    echo "::${level} title=${title}::${msg}"
-  fi
-}
-
-emit() {
-  local level="$1" file="$2" title="$3" msg="$4"
-  gh_annotate "$level" "$file" "$title" "$msg"
-  local upper
-  upper="$(echo "$level" | tr '[:lower:]' '[:upper:]')"
-  echo "${upper}: ${msg}" >&2
-  if [[ "$level" == "error" ]]; then
-    (( errors++ )) || true
-  else
-    (( warnings++ )) || true
-  fi
-}
-
-# Extract YAML frontmatter (text between first two --- lines, exclusive).
-# Prints frontmatter to stdout. Returns 1 if no valid frontmatter found.
-extract_frontmatter() {
-  local file="$1"
-  local first_line
-  first_line="$(head -n1 "$file" | sed 's/^[[:space:]]*//;s/[[:space:]]*$//')"
-  if [[ "$first_line" != "---" ]]; then
-    return 1
-  fi
-  # Find the closing --- (skip line 1, start from line 2)
-  local line_num=0
-  local found=0
-  while IFS= read -r line; do
-    line_num=$((line_num + 1))
-    if [[ $line_num -eq 1 ]]; then
-      continue
-    fi
-    local trimmed
-    trimmed="$(echo "$line" | sed 's/^[[:space:]]*//;s/[[:space:]]*$//')"
-    if [[ "$trimmed" == "---" ]]; then
-      found=1
-      break
-    fi
-    echo "$line"
-  done < "$file"
-  if [[ $found -eq 0 ]]; then
-    return 1
-  fi
-  return 0
-}
-
-# Extract keys from frontmatter text (stdin).
-# Outputs one key per line.
-frontmatter_keys() {
-  while IFS= read -r raw; do
-    local line
-    line="$(echo "$raw" | sed 's/^[[:space:]]*//;s/[[:space:]]*$//')"
-    # skip blank lines and comments
-    [[ -z "$line" ]] && continue
-    [[ "$line" == \#* ]] && continue
-    # must contain a colon
-    [[ "$line" != *:* ]] && continue
-    # extract key (everything before first colon), trimmed
-    local key
-    key="$(echo "$line" | cut -d: -f1 | sed 's/^[[:space:]]*//;s/[[:space:]]*$//')"
-    echo "$key"
-  done
-}
-
-# Check if a file contains an exact full line matching the needle.
-has_exact_line() {
-  local file="$1" needle="$2"
-  grep -qFx "$needle" "$file"
-}
-
-# ---------------------------------------------------------------------------
-# Config loading
-# ---------------------------------------------------------------------------
-
-load_config() {
-  local config_rel="${HARNESS_DOCS_CONFIG:-$DEFAULT_CONFIG_PATH}"
-  local config_path="$REPO_ROOT/$config_rel"
-  if [[ ! -f "$config_path" ]]; then
-    echo "Error: he-spikes-lint missing/invalid config: Missing config '${config_rel}'. Fix: create it (bootstrap should do this) or set HARNESS_DOCS_CONFIG." >&2
-    exit 2
-  fi
-  # Validate it is a JSON object
-  if ! jq -e 'type == "object"' "$config_path" >/dev/null 2>&1; then
-    echo "Error: he-spikes-lint missing/invalid config: Config must be a JSON object." >&2
-    exit 2
-  fi
-  CONFIG_PATH="$config_path"
-}
-
-# Read a JSON array from config as newline-delimited strings.
-config_string_array() {
-  local key="$1"
-  jq -r "(.${key} // []) | if type == \"array\" then .[] else empty end" "$CONFIG_PATH" 2>/dev/null | while IFS= read -r v; do
-    # only emit strings
-    echo "$v"
-  done
-}
-
-# ---------------------------------------------------------------------------
-# Per-spike checks
-# ---------------------------------------------------------------------------
-
-check_placeholders() {
-  local rel="$1" file="$2" fail_ph="$3"
-  shift 3
-  local patterns=("$@")
-  for p in "${patterns[@]}"; do
-    [[ -z "$p" ]] && continue
-    if grep -qF "$p" "$file"; then
-      local msg="Spike '${rel}' contains placeholder token '${p}'."
-      if [[ "$fail_ph" == "1" ]]; then
-        emit "error" "$rel" "Placeholder token" "$msg"
-      else
-        emit "warning" "$rel" "Placeholder token" "${msg} (Set HARNESS_FAIL_ON_ARTIFACT_PLACEHOLDERS=1 to enforce.)"
-      fi
-      break
-    fi
-  done
-}
-
-check_spike() {
-  local file="$1"
-  local rel="${file#"$REPO_ROOT"/}"
-
-  # --- frontmatter ---
-  local fm
-  if ! fm="$(extract_frontmatter "$file")"; then
-    emit "error" "$rel" "Missing YAML frontmatter" \
-      "Spike '${rel}' must start with YAML frontmatter delimited by '---' lines."
-    return
-  fi
-
-  # Check required frontmatter keys
-  local fm_keys
-  fm_keys="$(echo "$fm" | frontmatter_keys)"
-
-  local required_keys
-  required_keys="$(config_string_array "required_spike_frontmatter_keys")"
-
-  if [[ -n "$required_keys" ]]; then
-    while IFS= read -r k; do
-      [[ -z "$k" ]] && continue
-      if ! echo "$fm_keys" | grep -qFx "$k"; then
-        emit "error" "$rel" "Missing frontmatter key" \
-          "Spike '${rel}' missing YAML frontmatter key '${k}:'."
-      fi
-    done <<< "$required_keys"
-  fi
-
-  # --- required headings ---
-  for h in "${DEFAULT_REQUIRED_HEADINGS[@]}"; do
-    if ! has_exact_line "$file" "$h"; then
-      emit "error" "$rel" "Missing heading" \
-        "Spike '${rel}' missing required heading line '${h}'."
-    fi
-  done
-
-  # --- placeholder tokens ---
-  local placeholder_patterns=()
-  while IFS= read -r p; do
-    [[ -z "$p" ]] && continue
-    placeholder_patterns+=("$p")
-  done < <(config_string_array "artifact_placeholder_patterns")
-
-  local fail_ph="${HARNESS_FAIL_ON_ARTIFACT_PLACEHOLDERS:-0}"
-  if [[ ${#placeholder_patterns[@]} -gt 0 ]]; then
-    check_placeholders "$rel" "$file" "$fail_ph" "${placeholder_patterns[@]}"
-  fi
-}
-
-# ---------------------------------------------------------------------------
-# Main
-# ---------------------------------------------------------------------------
-
-load_config
-
-echo "he-spikes-lint: starting"
-echo "Repro: bash scripts/ci/he-spikes-lint.sh"
-
-spikes_dir="$REPO_ROOT/docs/spikes"
-if [[ ! -d "$spikes_dir" ]]; then
-  echo "he-spikes-lint: OK (docs/spikes not present)"
-  exit 0
-fi
-
-# Collect spike files sorted
-spike_files=()
-while IFS= read -r -d '' f; do
-  spike_files+=("$f")
-done < <(find "$spikes_dir" -maxdepth 1 -name '*-spike.md' -print0 | sort -z)
-
-if [[ ${#spike_files[@]} -eq 0 ]]; then
-  echo "he-spikes-lint: OK (no spike files)"
-  exit 0
-fi
-
-for f in "${spike_files[@]}"; do
-  check_spike "$f"
-done
-
-if [[ $errors -gt 0 ]]; then
-  echo "he-spikes-lint: FAIL (${errors} error(s), ${warnings} warning(s))" >&2
-  exit 1
-fi
-
-echo "he-spikes-lint: OK (${warnings} warning(s))"
-exit 0

package/scripts/runbooks/select-runbooks.sh

@@ -1,154 +0,0 @@
-#!/bin/bash
-set -euo pipefail
-
-# Select runbooks whose called_from frontmatter matches a skill or step name.
-# Prints matching runbook paths (relative to repo root) to stdout.
-
-# --- Repo root: two parents up from this script's directory ---
-SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
-REPO_ROOT="$(cd "$SCRIPT_DIR/../.." && pwd)"
-
-# --- CLI argument parsing ---
-SKILL=""
-STEP=""
-
-while [[ $# -gt 0 ]]; do
-  case "$1" in
-    --skill)
-      SKILL="$2"
-      shift 2
-      ;;
-    --step)
-      STEP="$2"
-      shift 2
-      ;;
-    *)
-      echo "Usage: $0 --skill <name> [--step <name>]" >&2
-      exit 1
-      ;;
-  esac
-done
-
-if [[ -z "$SKILL" ]]; then
-  echo "Error: --skill is required" >&2
-  exit 1
-fi
-
-# --- Main logic ---
-RUNBOOKS_DIR="$REPO_ROOT/docs/runbooks"
-
-if [[ ! -d "$RUNBOOKS_DIR" ]]; then
-  exit 0
-fi
-
-# Extract the frontmatter block (between first --- and next ---).
-# Parse called_from entries. Print the file path if skill or step matches.
-process_file() {
-  local file="$1"
-  local in_frontmatter=0
-  local in_called_from=0
-  local first_line=1
-  local called_from_items=()
-
-  while IFS= read -r line || [[ -n "$line" ]]; do
-    local trimmed
-    trimmed="$(echo "$line" | sed 's/^[[:space:]]*//;s/[[:space:]]*$//')"
-
-    # First non-empty consideration: frontmatter must start at line 1 with ---
-    if [[ "$first_line" -eq 1 ]]; then
-      first_line=0
-      if [[ "$trimmed" == "---" ]]; then
-        in_frontmatter=1
-        continue
-      else
-        # No frontmatter
-        return
-      fi
-    fi
-
-    # Inside frontmatter
-    if [[ "$in_frontmatter" -eq 1 ]]; then
-      # Closing delimiter
-      if [[ "$trimmed" == "---" ]]; then
-        break
-      fi
-
-      # Skip empty lines and comments
-      if [[ -z "$trimmed" || "$trimmed" == \#* ]]; then
-        # Empty lines inside a YAML list block: keep scanning
-        if [[ "$in_called_from" -eq 1 && -z "$trimmed" ]]; then
-          continue
-        fi
-        continue
-      fi
-
-      # If we're collecting YAML list items for called_from
-      if [[ "$in_called_from" -eq 1 ]]; then
-        # Check if this is a list item (starts with -)
-        if [[ "$trimmed" == -* ]]; then
-          local item
-          item="$(echo "$trimmed" | sed "s/^-[[:space:]]*//;s/^[\"']//;s/[\"']$//")"
-          if [[ -n "$item" ]]; then
-            called_from_items+=("$item")
-          fi
-          continue
-        else
-          # Not a list item; if it contains a colon it's a new key — stop collecting
-          if echo "$trimmed" | grep -q ':'; then
-            in_called_from=0
-            # Fall through to process this line as a new key
-          else
-            continue
-          fi
-        fi
-      fi
-
-      # Check for key: value lines
-      if echo "$trimmed" | grep -q ':'; then
-        local key val
-        key="$(echo "$trimmed" | sed 's/^\([^:]*\):.*/\1/' | sed 's/^[[:space:]]*//;s/[[:space:]]*$//')"
-        val="$(echo "$trimmed" | sed 's/^[^:]*://' | sed 's/^[[:space:]]*//;s/[[:space:]]*$//')"
-
-        if [[ "$key" == "called_from" ]]; then
-          # Inline list: called_from: [a, b]
-          if [[ "$val" == \[* ]]; then
-            local inner
-            inner="$(echo "$val" | sed 's/^\[//;s/\]$//' | sed 's/^[[:space:]]*//;s/[[:space:]]*$//')"
-            if [[ -n "$inner" ]]; then
-              IFS=',' read -ra parts <<< "$inner"
-              for part in "${parts[@]}"; do
-                local cleaned
-                cleaned="$(echo "$part" | sed 's/^[[:space:]]*//;s/[[:space:]]*$//')"
-                if [[ -n "$cleaned" ]]; then
-                  called_from_items+=("$cleaned")
-                fi
-              done
-            fi
-          else
-            # YAML list form — start collecting on subsequent lines
-            in_called_from=1
-          fi
-        fi
-      fi
-    fi
-  done < "$file"
-
-  # Check for matches
-  for item in "${called_from_items[@]+"${called_from_items[@]}"}"; do
-    if [[ "$item" == "$SKILL" ]]; then
-      echo "${file#"$REPO_ROOT"/}"
-      return
-    fi
-    if [[ -n "$STEP" && "$item" == "$STEP" ]]; then
-      echo "${file#"$REPO_ROOT"/}"
-      return
-    fi
-  done
-}
-
-# Find all .md files, sorted for deterministic output
-while IFS= read -r mdfile; do
-  process_file "$mdfile"
-done < <(find "$RUNBOOKS_DIR" -name '*.md' -type f | sort)
-
-exit 0

package/src/config.ts

@@ -1,183 +0,0 @@
-import {
-  canonicalType,
-  type EntityAllowlist,
-  type FogClawConfig,
-  type GuardrailAction,
-  type RedactStrategy,
-} from "./types.js";
-
-const VALID_GUARDRAIL_MODES: GuardrailAction[] = ["redact", "block", "warn"];
-const VALID_REDACT_STRATEGIES: RedactStrategy[] = ["token", "mask", "hash"];
-
-function ensureStringList(value: unknown, path: string): string[] {
-  if (!Array.isArray(value)) {
-    throw new Error(`${path} must be an array of strings`);
-  }
-
-  const entries = value.filter((entry): entry is string => {
-    if (typeof entry !== "string") {
-      throw new Error(`${path} must contain only strings`);
-    }
-
-    return true;
-  });
-
-  return entries.map((entry) => entry.trim()).filter((entry) => entry.length > 0);
-}
-
-function ensureEntityAllowlist(value: unknown): EntityAllowlist {
-  if (value == null) {
-    return { values: [], patterns: [], entities: {} };
-  }
-
-  if (typeof value !== "object" || Array.isArray(value)) {
-    throw new Error("allowlist must be an object");
-  }
-
-  const raw = value as Record<string, unknown>;
-  const values = ensureStringList(raw.values ?? [], "allowlist.values");
-  const patterns = ensureStringList(raw.patterns ?? [], "allowlist.patterns");
-
-  for (const pattern of patterns) {
-    try {
-      new RegExp(pattern);
-    } catch {
-      throw new Error(`allowlist.patterns contains invalid regex pattern: "${pattern}"`);
-    }
-  }
-
-  const entitiesValue = raw.entities ?? {};
-  if (
-    typeof entitiesValue !== "object" ||
-    Array.isArray(entitiesValue) ||
-    entitiesValue === null
-  ) {
-    throw new Error("allowlist.entities must be an object mapping entity labels to string arrays");
-  }
-
-  const entities: Record<string, string[]> = {};
-  for (const [entityType, entryValue] of Object.entries(entitiesValue)) {
-    const normalizedType = canonicalType(entityType);
-    entities[normalizedType] = ensureStringList(entryValue, `allowlist.entities.${entityType}`);
-  }
-
-  return {
-    values: [...new Set(values)],
-    patterns: [...new Set(patterns)],
-    entities,
-  };
-}
-
-function ensureEntityConfidenceThresholds(
-  value: unknown,
-): Record<string, number> {
-  if (!value) {
-    return {};
-  }
-
-  if (typeof value !== "object" || Array.isArray(value) || value === null) {
-    throw new Error("entityConfidenceThresholds must be an object");
-  }
-
-  const raw = value as Record<string, unknown>;
-  const normalized: Record<string, number> = {};
-
-  for (const [entityType, rawThreshold] of Object.entries(raw)) {
-    if (typeof rawThreshold !== "number" || Number.isNaN(rawThreshold)) {
-      throw new Error(
-        `entityConfidenceThresholds["${entityType}"] must be a number between 0 and 1, got ${String(
-          rawThreshold,
-        )}`,
-      );
-    }
-
-    if (rawThreshold < 0 || rawThreshold > 1) {
-      throw new Error(
-        `entityConfidenceThresholds["${entityType}"] must be between 0 and 1, got ${rawThreshold}`,
-      );
-    }
-
-    const canonicalTypeKey = canonicalType(entityType);
-    normalized[canonicalTypeKey] = rawThreshold;
-  }
-
-  return normalized;
-}
-
-export const DEFAULT_CONFIG: FogClawConfig = {
-  enabled: true,
-  guardrail_mode: "redact",
-  redactStrategy: "token",
-  model: "onnx-community/gliner_large-v2.1",
-  confidence_threshold: 0.5,
-  custom_entities: [],
-  entityActions: {},
-  entityConfidenceThresholds: {},
-  allowlist: {
-    values: [],
-    patterns: [],
-    entities: {},
-  },
-  auditEnabled: true,
-};
-
-export function loadConfig(overrides: Partial<FogClawConfig>): FogClawConfig {
-  const config: FogClawConfig = {
-    ...DEFAULT_CONFIG,
-    ...overrides,
-    entityActions: {
-      ...DEFAULT_CONFIG.entityActions,
-      ...(overrides.entityActions ?? {}),
-    },
-    entityConfidenceThresholds: {
-      ...DEFAULT_CONFIG.entityConfidenceThresholds,
-      ...(overrides.entityConfidenceThresholds ?? {}),
-    },
-  };
-
-  config.allowlist = ensureEntityAllowlist(overrides.allowlist ?? DEFAULT_CONFIG.allowlist);
-  config.entityConfidenceThresholds = ensureEntityConfidenceThresholds(
-    config.entityConfidenceThresholds,
-  );
-
-  if (typeof config.enabled !== "boolean") {
-    throw new Error(`enabled must be true or false`);
-  }
-
-  if (!VALID_GUARDRAIL_MODES.includes(config.guardrail_mode)) {
-    throw new Error(
-      `Invalid guardrail_mode "${config.guardrail_mode}". Must be one of: ${VALID_GUARDRAIL_MODES.join(", ")}`,
-    );
-  }
-
-  if (!VALID_REDACT_STRATEGIES.includes(config.redactStrategy)) {
-    throw new Error(
-      `Invalid redactStrategy "${config.redactStrategy}". Must be one of: ${VALID_REDACT_STRATEGIES.join(", ")}`,
-    );
-  }
-
-  if (config.confidence_threshold < 0 || config.confidence_threshold > 1) {
-    throw new Error(
-      `confidence_threshold must be between 0 and 1, got ${config.confidence_threshold}`,
-    );
-  }
-
-  if (typeof config.auditEnabled !== "boolean") {
-    throw new Error(`auditEnabled must be true or false`);
-  }
-
-  const normalizedActions: Record<string, GuardrailAction> = {};
-  for (const [entityType, action] of Object.entries(config.entityActions)) {
-    if (!VALID_GUARDRAIL_MODES.includes(action)) {
-      throw new Error(
-        `Invalid action "${action}" for entity type "${entityType}". Must be one of: ${VALID_GUARDRAIL_MODES.join(", ")}`,
-      );
-    }
-
-    const normalizedType = canonicalType(entityType);
-    normalizedActions[normalizedType] = action;
-  }
-  config.entityActions = normalizedActions;
-
-  return config;
-}