@djordje-stojanovic/sigmaskills 0.2.0

package/src/registry/normalize.js

@@ -0,0 +1,67 @@
+'use strict';
+
+/**
+ * normalize.js — turn parsed upstream host records into the canonical,
+ * versioned, attributed registry snapshot consumed by the installer.
+ *
+ * Only declarative data survives normalization. Detection function bodies
+ * are reduced to env-var metadata; destination formulas are kept as safe
+ * declarative records (kind: literal | join | conditional | none). Nothing
+ * is evaluated. Hosts are sorted bytewise by id so the emitted JSON is
+ * reproducible regardless of host locale or ICU settings.
+ */
+
+/**
+ * Bytewise (code-point) id comparator, locale-independent. Guarantees the
+ * snapshot ordering is identical on every machine and Node build.
+ */
+export function compareIds(a, b) {
+  return a < b ? -1 : a > b ? 1 : 0;
+}
+
+/**
+ * Normalize a parsed host record into a canonical registry entry.
+ * Adds attribution (upstream file + line) and a stable normalized shape.
+ */
+export function normalizeHost(host, source) {
+  return {
+    id: host.id,
+    name: host.name,
+    displayName: host.displayName,
+    universal: host.universal,
+    universalPrompt: host.universalPrompt,
+    destinations: {
+      project: host.project ?? { kind: 'none', raw: undefined },
+      global: host.global ?? { kind: 'none', raw: undefined },
+    },
+    aliases: host.aliases ?? [],
+    platforms: host.platforms ?? [],
+    detection: {
+      envVars: host.envVars ?? [],
+    },
+    attribution: {
+      upstreamFile: source.upstreamFile ?? source.file,
+      upstreamLine: host.line,
+    },
+  };
+}
+
+/**
+ * Build the canonical snapshot container.
+ * @param {Array} hosts  normalized host records
+ * @param {object} source pin record { repository, pinnedRevision, upstreamFile, fetchedAt? }
+ */
+export function buildSnapshot(hosts, source) {
+  const sorted = [...hosts].sort((a, b) => compareIds(a.id, b.id));
+  const generatedFrom = {
+    repository: source.repository,
+    pinnedRevision: source.pinnedRevision ?? source.revision,
+    upstreamFile: source.upstreamFile ?? source.file,
+  };
+  if (source.fetchedAt) generatedFrom.fetchedAt = source.fetchedAt;
+  return {
+    schemaVersion: 1,
+    generatedFrom,
+    hosts: sorted,
+  };
+}

package/src/registry/parse.js

@@ -0,0 +1,184 @@
+'use strict';
+
+/**
+ * parse.js — extract declarative Agent Host data from upstream registry text.
+ *
+ * Sigma never executes upstream TypeScript, detector, or path-building code.
+ * This parser reads upstream `agents.ts` as untrusted text and extracts only
+ * literal, declarative fields: host ids, display names, project/global
+ * destination formulas, aliases, platform declarations, and env-var
+ * detection metadata. Conditional helpers are extracted as declarative
+ * case lists (probe + formula), never evaluated.
+ */
+
+/** Env-var names referenced in a text fragment (metadata, never evaluated). */
+export function extractEnvVars(text) {
+  const names = new Set();
+  const re = /(?:env|process\.env)\.([A-Z][A-Z0-9_]*)/g;
+  let m;
+  while ((m = re.exec(text)) !== null) names.add(m[1]);
+  return [...names].sort();
+}
+
+/** Capture the value of a single-line field like `  skillsDir: '...',`. */
+function field(body, name) {
+  const re = new RegExp('^\\s*' + name + '\\s*:\\s*([^\\n]+)', 'm');
+  const m = re.exec(body);
+  if (!m) return null;
+  let value = m[1].trim();
+  value = value.replace(/,$/, '').trim();
+  return value;
+}
+
+/** Unquote a single-quoted upstream literal. Returns null when malformed. */
+function quoted(value) {
+  const m = /^'((?:[^'\\]|\\.)*)'$/.exec(value.trim());
+  return m ? m[1] : null;
+}
+
+const JOIN_RE = /^join\(([A-Za-z0-9_.]+)((?:,\s*'((?:[^'\\]|\\.)*)')*)\)$/;
+
+/** Parse a destination formula. Forms: join(...) | '<path>' | undefined | <fn>(). */
+export function parseDirFormula(value) {
+  const raw = value.trim();
+  if (!raw) return null;
+  if (raw === 'undefined') return { kind: 'none', raw };
+
+  const quotedPath = quoted(raw);
+  if (quotedPath !== null) return { kind: 'literal', path: quotedPath, raw };
+
+  const join = JOIN_RE.exec(raw);
+  if (join) {
+    const segmentsRe = /'((?:[^'\\]|\\.)*)'/g;
+    const segments = [];
+    let sm;
+    while ((sm = segmentsRe.exec(raw)) !== null) segments.push(sm[1]);
+    return { kind: 'join', base: join[1], segments, raw };
+  }
+
+  const fnRe = /^([A-Za-z0-9_]+)\(\)$/;
+  const fn = fnRe.exec(raw);
+  if (fn) return { kind: 'function', name: fn[1], raw };
+
+  return { kind: 'unknown', raw };
+}
+
+const PROBE_RE = /(?:existsSync|pathExists)\(join\(([A-Za-z0-9_.]+)\s*,\s*'([^']*)'\)\)/g;
+const RETURN_RE = /return\s+(join\([^;]*\)|[^;\n]+);/g;
+
+/**
+ * Declaratively extract a conditional helper `function <name>(...) { ... }`.
+ * Returns { kind: 'conditional', base, cases: [{ probe, formula }] } where
+ * cases are (probe, return-join) pairs and the final return is the default.
+ * Guards and returns are parsed as text; nothing is evaluated.
+ */
+export function parseConditionalHelper(sourceText, name) {
+  const anchor = 'function ' + name;
+  let idx = sourceText.indexOf(anchor);
+  if (idx === -1) return null;
+  let p = sourceText.indexOf('(', idx + anchor.length);
+  if (p === -1) return null;
+  let depth = 1;
+  let q = p + 1;
+  for (; q < sourceText.length; q++) {
+    const ch = sourceText[q];
+    if (ch === '(') depth++;
+    else if (ch === ')') { depth--; if (depth === 0) break; }
+  }
+  if (depth !== 0) return null;
+  let brace = sourceText.indexOf('{', q);
+  if (brace === -1) return null;
+  const bodyStart = brace + 1;
+  let braceDepth = 1;
+  let i = bodyStart;
+  for (; i < sourceText.length; i++) {
+    const ch = sourceText[i];
+    if (ch === '{') braceDepth++;
+    else if (ch === '}') {
+      braceDepth--;
+      if (braceDepth === 0) break;
+    }
+  }
+  const body = sourceText.slice(bodyStart, i);
+
+  const probes = [];
+  let pm;
+  while ((pm = PROBE_RE.exec(body)) !== null) {
+    probes.push({ index: pm.index, end: pm.index + pm[0].length, probe: pm[2] });
+  }
+
+  const cases = [];
+  let rm;
+  while ((rm = RETURN_RE.exec(body)) !== null) {
+    let probe = null;
+    for (const p of probes) {
+      if (p.end > rm.index) break;
+      const between = body.slice(p.end, rm.index);
+      if (!between.includes('}')) probe = p.probe;
+    }
+    const formula = parseDirFormula(rm[1]);
+    cases.push({ probe, formula });
+  }
+  if (cases.length === 0) return null;
+  const first = cases.find((c) => c.formula && c.formula.base);
+  const base = first ? first.formula.base : null;
+  return { kind: 'conditional', base, cases };
+}
+
+/** Boolean value of an upstream literal boolean field. */
+function bool(value) {
+  if (value === null) return null;
+  if (value === 'true') return true;
+  if (value === 'false') return false;
+  return null;
+}
+
+/** Parse one host entry body into a declarative record. */
+function parseHostBody(id, body, line, sourceText) {
+  const name = quoted(field(body, 'name')) ?? id;
+  const displayName = quoted(field(body, 'displayName')) ?? name;
+  const project = parseDirFormula(field(body, 'skillsDir'));
+  const globalRaw = field(body, 'globalSkillsDir');
+  let global = parseDirFormula(globalRaw);
+  if (global && global.kind === 'function') {
+    const conditional = parseConditionalHelper(sourceText, global.name);
+    if (conditional) global = conditional;
+  }
+  const aliasesRaw = field(body, 'aliases');
+  const aliases = aliasesRaw ? aliasesRaw.replace(/^\[|\]$/g, '').split(',').map((s) => s.trim()) : [];
+  const platformsRaw = field(body, 'platforms');
+  const platforms = platformsRaw ? platformsRaw.replace(/^\[|\]$/g, '').split(',').map((s) => s.trim()) : [];
+  const showInUniversalList = bool(field(body, 'showInUniversalList'));
+  const showInUniversalPrompt = bool(field(body, 'showInUniversalPrompt'));
+  const envVars = extractEnvVars(body);
+
+  return {
+    id,
+    name,
+    displayName,
+    project,
+    global,
+    aliases,
+    platforms,
+    universal: showInUniversalList !== false,
+    universalPrompt: showInUniversalPrompt !== false,
+    envVars,
+    line,
+  };
+}
+
+/**
+ * Parse upstream agents.ts text into declarative host records.
+ */
+export function parseAgents(agentsText) {
+  const hosts = [];
+  const entryRe = /^  '?([A-Za-z0-9_-]+)'?: \{([\s\S]*?)\n  \},$/gm;
+  let m;
+  while ((m = entryRe.exec(agentsText)) !== null) {
+    const id = m[1];
+    const body = m[2];
+    const line = 1 + agentsText.slice(0, m.index).split('\n').length;
+    hosts.push(parseHostBody(id, body, line, agentsText));
+  }
+  return hosts;
+}

package/src/registry/sync.js

@@ -0,0 +1,230 @@
+'use strict';
+
+/**
+ * sync.js — deterministic registry synchronization command.
+ *
+ *   node src/registry/sync.js [--fetch] [--dry-run] [--allow-review]
+ *
+ * Reads upstream `agents.ts` (the pinned revision recorded in
+ * `registry/source.json`), verifies its content hash against the recorded
+ * pin, parses only declarative fields, normalizes into the canonical
+ * attributed snapshot, validates, and emits a normalized semantic diff
+ * against the existing snapshot. Unless --dry-run, writes the snapshot only
+ * when there are no review-classified changes, or when --allow-review is
+ * passed. `--fetch` downloads the pinned revision from the upstream
+ * repository and saves it as the local pinned fixture before syncing.
+ *
+ * The default pin is the real immutable upstream commit recorded in
+ * `registry/source.json`; a pseudo-pin is rejected so a release can never
+ * masquerade as synchronized from upstream.
+ */
+
+import crypto from 'node:crypto';
+import fs from 'node:fs';
+import path from 'node:path';
+import { fileURLToPath } from 'node:url';
+import { parseAgents } from './parse.js';
+import { normalizeHost, buildSnapshot } from './normalize.js';
+import { validateSnapshot } from './validate.js';
+import { diffSnapshots } from './diff.js';
+
+const __dirname = path.dirname(fileURLToPath(import.meta.url));
+const ROOT = path.resolve(__dirname, '..', '..');
+const REGISTRY_DIR = path.join(ROOT, 'registry');
+const FIXTURE_PATH = path.join(ROOT, 'test', 'fixtures', 'vercel-skills', 'src', 'agents.ts');
+const PIN_PATH = path.join(REGISTRY_DIR, 'source.json');
+const SNAPSHOT_PATH = path.join(REGISTRY_DIR, 'agent-hosts.json');
+
+const REVISION_PATTERN = /^[0-9a-f]{40}$/;
+const SHA256_PATTERN = /^[0-9a-f]{64}$/;
+
+function sha256(text) {
+  return crypto.createHash('sha256').update(text, 'utf8').digest('hex');
+}
+
+export function canonicalSourceText(text) {
+  return String(text || '').replace(/\r\n/g, '\n').replace(/\r/g, '\n');
+}
+
+function parseArgs(argv) {
+  const args = { fetch: false, dryRun: false, allowReview: false, source: null };
+  for (let i = 0; i < argv.length; i++) {
+    const arg = argv[i];
+    if (arg === '--dry-run') args.dryRun = true;
+    else if (arg === '--allow-review') args.allowReview = true;
+    else if (arg === '--fetch') args.fetch = true;
+    else if (arg === '--source' && i + 1 < argv.length) args.source = argv[++i];
+  }
+  return args;
+}
+
+function readJson(file) {
+  if (!fs.existsSync(file)) return null;
+  return JSON.parse(fs.readFileSync(file, 'utf8'));
+}
+
+function writeJson(file, value) {
+  fs.mkdirSync(path.dirname(file), { recursive: true });
+  fs.writeFileSync(file, JSON.stringify(value, null, 2) + '\n', 'utf8');
+}
+
+/**
+ * Load the pinned source record and reject pseudo-pins.
+ * @returns {object} pin { repository, pinnedRevision, upstreamFile, contentSha256 }
+ */
+export function loadPin(pinPath = PIN_PATH) {
+  const pin = readJson(pinPath);
+  if (!pin || !pin.pinnedRevision || !REVISION_PATTERN.test(pin.pinnedRevision)) {
+    throw new Error(
+      `registry/source.json must pin an immutable 40-hex upstream revision; got ${JSON.stringify(pin && pin.pinnedRevision)}`
+    );
+  }
+  if (typeof pin.repository !== 'string' || pin.repository.length === 0) {
+    throw new Error('registry/source.json must record the upstream repository URL');
+  }
+  if (!pin.upstreamFile || !pin.contentSha256 || !SHA256_PATTERN.test(pin.contentSha256)) {
+    throw new Error('registry/source.json must record the upstream file path and contentSha256');
+  }
+  return pin;
+}
+
+/**
+ * Fetch the pinned upstream file from the repository at the pinned revision
+ * and save it as the local pinned fixture. Verifies the content hash.
+ * @param {object} pin loaded pin record
+ * @param {string} [fixturePath] where to save the fetched content
+ */
+export async function fetchPinnedSource(pin, fixturePath = FIXTURE_PATH) {
+  const url = `https://raw.githubusercontent.com/${pin.repository}/${pin.pinnedRevision}/${pin.upstreamFile}`;
+  let res;
+  try {
+    res = await fetch(url);
+  } catch (err) {
+    throw new Error(`failed to fetch ${url}: ${err.message}`);
+  }
+  if (!res.ok) {
+    throw new Error(`failed to fetch ${url}: HTTP ${res.status}`);
+  }
+  const text = canonicalSourceText(await res.text());
+  const digest = sha256(text);
+  if (digest !== pin.contentSha256) {
+    throw new Error(
+      `fetched content hash ${digest} does not match pinned contentSha256 ${pin.contentSha256} for ${pin.pinnedRevision}`
+    );
+  }
+  fs.mkdirSync(path.dirname(fixturePath), { recursive: true });
+  fs.writeFileSync(fixturePath, text, 'utf8');
+  return text;
+}
+
+/**
+ * Synchronize the registry from the pinned upstream source.
+ *
+ * @param {object} args { fetch, dryRun, allowReview, source? }
+ * @param {object} [options] { pin?, previous?, fixturePath? }
+ * @returns {object} { ok, snapshot, validation, diff, shouldWrite, errors? }
+ */
+export function syncRegistry(args, options = {}) {
+  const pin = options.pin ?? loadPin();
+  const fixturePath = options.fixturePath ?? FIXTURE_PATH;
+  const sourcePath = args.source ?? fixturePath;
+
+  let sourceText;
+  if (args.fetch) {
+    sourceText = fetchPinnedSourceSync(pin, fixturePath);
+  } else {
+    sourceText = canonicalSourceText(fs.readFileSync(sourcePath, 'utf8'));
+    const digest = sha256(sourceText);
+    if (digest !== pin.contentSha256) {
+      return {
+        ok: false,
+        errors: [
+          `source content hash ${digest} does not match pinned contentSha256 ${pin.contentSha256}` +
+            ` (source ${sourcePath}); re-fetch with --fetch or update registry/source.json`,
+        ],
+      };
+    }
+  }
+
+  const parsed = parseAgents(sourceText);
+  const hosts = parsed.map((h) => normalizeHost(h, pin));
+  const snapshot = buildSnapshot(hosts, pin);
+
+  const validation = validateSnapshot(snapshot);
+  if (!validation.valid) {
+    return {
+      ok: false,
+      errors: [...validation.errors, ...Object.values(validation.hostErrors)],
+    };
+  }
+
+  const previous = options.previous ?? readJson(SNAPSHOT_PATH);
+  const diff = previous ? diffSnapshots(previous, snapshot) : null;
+  const reviewCount = diff ? diff.summary.review : 0;
+
+  const shouldWrite = !args.dryRun && (args.allowReview || reviewCount === 0);
+
+  if (shouldWrite) {
+    writeJson(SNAPSHOT_PATH, snapshot);
+    writeJson(PIN_PATH, pin);
+  }
+
+  return {
+    ok: true,
+    snapshot,
+    validation,
+    diff,
+    shouldWrite,
+  };
+}
+
+// Small sync wrapper so syncRegistry stays pure for tests.
+function fetchPinnedSourceSync(pin, fixturePath) {
+  const url = `https://raw.githubusercontent.com/${pin.repository}/${pin.pinnedRevision}/${pin.upstreamFile}`;
+  throw new Error(`sync --fetch requires async fetch; use runSync() instead (${url})`);
+}
+
+/**
+ * Async entrypoint used by the CLI. Fetches when requested, then syncs.
+ * @param {object} args CLI args
+ * @returns {Promise<object>} result of syncRegistry
+ */
+export async function runSync(args, options = {}) {
+  const pin = options.pin ?? loadPin();
+  if (args.fetch) {
+    await fetchPinnedSource(pin, options.fixturePath ?? FIXTURE_PATH);
+  }
+  return syncRegistry({ ...args, fetch: false }, options);
+}
+
+function main() {
+  const args = parseArgs(process.argv.slice(2));
+  runSync(args).then((result) => {
+    if (!result.ok) {
+      for (const err of result.errors) console.error('validation error:', err);
+      process.exit(1);
+    }
+    console.log(`sync: parsed ${result.snapshot.hosts.length} hosts from pinned revision ${loadPin().pinnedRevision}`);
+    if (result.diff) {
+      console.log('diff summary:', JSON.stringify(result.diff.summary));
+      for (const change of result.diff.changes) {
+        console.log(`  ${change.authority.padEnd(6)} ${change.id} ${change.kind}` +
+          (change.fields ? ' [' + change.fields.join(', ') + ']' : ''));
+      }
+    }
+    if (result.shouldWrite) {
+      console.log('wrote registry/agent-hosts.json + registry/source.json');
+    } else if (args.dryRun) {
+      console.log('dry run: no files written');
+    } else {
+      console.log('review-classified changes present; registry not written (use --allow-review to force)');
+    }
+  }).catch((err) => {
+    console.error('error:', err.message);
+    process.exit(1);
+  });
+}
+
+if (process.argv[1] && path.basename(process.argv[1]) === 'sync.js') {
+  main();
+}