@elmundi/ship-cli 0.8.1 → 0.11.2

package/lib/state/idempotency.mjs

@@ -0,0 +1,183 @@
+/**
+ * Idempotency markers for `kind=once` lanes (RFC-0007).
+ *
+ * Markers live under `.ship/state/<key>.json` and are expected to be
+ * committed to the repo. They record that a particular one-shot lane
+ * has already run for a specific version of its pattern. The agent
+ * orchestration layer reads them before deciding whether to execute the
+ * lane again and writes them on success.
+ *
+ * File format (version 1):
+ *
+ *   {
+ *     "version": 1,
+ *     "lane": "seed_knowledge_starters",
+ *     "pattern_id": "seed-knowledge-starters",
+ *     "pattern_sha256": "…",
+ *     "pattern_version": "1.0.0",
+ *     "completed_at": "2026-04-21T14:20:00Z",
+ *     "by": { "run_id": "…", "host": "github-actions" }
+ *   }
+ *
+ * We keep the reader tolerant of extra keys (forward-compat) and the
+ * writer strict about the required ones (backward-compat).
+ */
+
+import fs from "node:fs";
+import path from "node:path";
+import { createHash } from "node:crypto";
+
+import { findShipRoot } from "../config/io.mjs";
+
+export const MARKER_SCHEMA_VERSION = 1;
+export const STATE_SUBDIR = path.join(".ship", "state");
+
+const KEY_REGEX = /^[a-z0-9][a-z0-9_.-]{0,127}$/;
+
+/**
+ * Resolve the marker path for a given idempotency key. Does not check
+ * existence — callers use {@link readMarker} for that.
+ *
+ * @param {string} cwd
+ * @param {string} key
+ * @returns {{ root: string, markerPath: string }}
+ */
+export function resolveMarkerPath(cwd, key) {
+  if (typeof key !== "string" || !KEY_REGEX.test(key)) {
+    throw new Error(
+      `idempotency key must match /^[a-z0-9][a-z0-9_.-]{0,127}$/; got ${JSON.stringify(key)}`,
+    );
+  }
+  const root = findShipRoot(cwd);
+  if (!root) {
+    throw new Error(
+      ".ship/config.yml not found; idempotency markers require a Ship-adopted repo",
+    );
+  }
+  return {
+    root,
+    markerPath: path.join(root, STATE_SUBDIR, `${key}.json`),
+  };
+}
+
+/**
+ * Read and validate an idempotency marker. Returns `null` when the
+ * marker file doesn't exist; throws on malformed JSON or schema
+ * mismatch (callers should decide whether to treat that as fatal —
+ * `shipctl run` currently treats it as fatal so a broken marker can't
+ * silently cause a re-seed).
+ *
+ * @param {string} cwd
+ * @param {string} key
+ * @returns {object | null}
+ */
+export function readMarker(cwd, key) {
+  const { markerPath } = resolveMarkerPath(cwd, key);
+  if (!fs.existsSync(markerPath)) return null;
+  let raw;
+  try {
+    raw = fs.readFileSync(markerPath, "utf8");
+  } catch (e) {
+    throw new Error(`idempotency: failed to read ${markerPath}: ${e.message}`);
+  }
+  let parsed;
+  try {
+    parsed = JSON.parse(raw);
+  } catch (e) {
+    throw new Error(`idempotency: ${markerPath} is not valid JSON: ${e.message}`);
+  }
+  if (!parsed || typeof parsed !== "object" || Array.isArray(parsed)) {
+    throw new Error(`idempotency: ${markerPath} must be a JSON object`);
+  }
+  if (parsed.version !== MARKER_SCHEMA_VERSION) {
+    throw new Error(
+      `idempotency: ${markerPath} has marker version ${JSON.stringify(parsed.version)}; expected ${MARKER_SCHEMA_VERSION}`,
+    );
+  }
+  return parsed;
+}
+
+/**
+ * Write an idempotency marker atomically (tmp file + rename). Creates
+ * `.ship/state/` if missing.
+ *
+ * @param {string} cwd
+ * @param {string} key
+ * @param {{
+ *   lane: string,
+ *   pattern_id: string,
+ *   pattern_sha256: string,
+ *   pattern_version?: string | null,
+ *   by?: Record<string, unknown>,
+ *   completed_at?: string,
+ * }} fields
+ * @returns {{ markerPath: string, marker: object }}
+ */
+export function writeMarker(cwd, key, fields) {
+  const { markerPath } = resolveMarkerPath(cwd, key);
+  const marker = {
+    version: MARKER_SCHEMA_VERSION,
+    lane: String(fields.lane),
+    pattern_id: String(fields.pattern_id),
+    pattern_sha256: String(fields.pattern_sha256),
+    pattern_version: fields.pattern_version ?? null,
+    completed_at: fields.completed_at ?? new Date().toISOString(),
+    by: fields.by ?? defaultActorContext(),
+  };
+
+  const required = ["lane", "pattern_id", "pattern_sha256"];
+  for (const k of required) {
+    if (!marker[k]) throw new Error(`idempotency: missing required field '${k}'`);
+  }
+
+  fs.mkdirSync(path.dirname(markerPath), { recursive: true });
+  const body = `${JSON.stringify(marker, null, 2)}\n`;
+  const tmp = `${markerPath}.tmp`;
+  fs.writeFileSync(tmp, body, "utf8");
+  fs.renameSync(tmp, markerPath);
+  return { markerPath, marker };
+}
+
+/**
+ * Decide whether a `kind=once` lane should run given its current
+ * marker and the current pattern body.
+ *
+ * @param {object | null} marker      result of readMarker()
+ * @param {string} patternBody        full ARTIFACT.md text (including frontmatter)
+ * @param {"version-change" | "manual"} resetOn
+ * @returns {{ run: true, reason: "no-marker" | "sha-changed" } | { run: false, reason: "already-done", marker: object }}
+ */
+export function decideRun(marker, patternBody, resetOn) {
+  if (!marker) return { run: true, reason: "no-marker" };
+  const sha = sha256(patternBody);
+  if (resetOn === "version-change" && marker.pattern_sha256 !== sha) {
+    return { run: true, reason: "sha-changed" };
+  }
+  return { run: false, reason: "already-done", marker };
+}
+
+/**
+ * SHA-256 of the full pattern body (frontmatter included). Exposed so
+ * callers can recompute it when writing markers without re-parsing.
+ *
+ * @param {string} text
+ * @returns {string}
+ */
+export function sha256(text) {
+  return createHash("sha256").update(text, "utf8").digest("hex");
+}
+
+function defaultActorContext() {
+  /* Prefer the richest available CI context but keep it small — this
+   * blob ends up committed in the repo, so we don't want noise. */
+  const ctx = {};
+  if (process.env.GITHUB_ACTIONS) ctx.host = "github-actions";
+  else if (process.env.GITLAB_CI) ctx.host = "gitlab-ci";
+  else if (process.env.CI) ctx.host = "ci";
+  else ctx.host = "local";
+
+  if (process.env.SHIP_RUN_ID) ctx.run_id = process.env.SHIP_RUN_ID;
+  else if (process.env.GITHUB_RUN_ID) ctx.run_id = process.env.GITHUB_RUN_ID;
+
+  return ctx;
+}

package/lib/state/lockfile.mjs

@@ -0,0 +1,180 @@
+/**
+ * `shipctl.lock.json` — reproducible-build anchor for Ship lanes (RFC-0007
+ * Phase 4). Records the exact `(kind, id, version, content_sha256)` of
+ * every artifact a lane depends on, plus where that body is materialised on
+ * disk. Without it, `shipctl run --offline` has no way to decide whether
+ * a cached pattern is "the one the lane expects" — the lockfile gives a
+ * positive, auditable answer.
+ *
+ * Only patterns are locked today (lanes only reference patterns via
+ * `lane.pattern`). The schema is forward-compatible: tools and collections
+ * can be added to the same `artifacts` map with no version bump.
+ *
+ * Concurrency: the writer does an atomic rename (write to `.tmp` in the
+ * same directory, rename over the target). Readers always open a read-only
+ * handle so a race doesn't yield a truncated parse.
+ */
+
+import fs from "node:fs";
+import path from "node:path";
+
+import { artifactSha256 } from "../cache/store.mjs";
+
+const LOCKFILE_REL = path.join(".ship", "shipctl.lock.json");
+export const LOCKFILE_SCHEMA_VERSION = 1;
+
+export function lockfilePath(shipRoot) {
+  return path.join(shipRoot, LOCKFILE_REL);
+}
+
+/**
+ * @typedef {Object} LockfileEntry
+ * @property {string} version             Resolved version of the artifact.
+ * @property {string} content_sha256      Hex digest (RFC-0005 normalisation).
+ * @property {string} cached_path         Relative path inside the ship root.
+ * @property {"http" | "monorepo" | "inline"} source
+ * @property {boolean} pinned             Whether a config pin constrained this.
+ * @property {string} [channel]           Manifest channel at time of lock.
+ *
+ * @typedef {Object} Lockfile
+ * @property {1} version
+ * @property {string} generated_at        ISO-8601 UTC.
+ * @property {string} shipctl_version
+ * @property {{ base_url: string, channel: string } | null} source
+ * @property {Record<string, LockfileEntry>} artifacts  Key: "<kind>/<id>".
+ * @property {string[]} [notes]           Free-form operator hints.
+ */
+
+/**
+ * @param {string} shipRoot
+ * @returns {Lockfile | null}
+ */
+export function readLockfile(shipRoot) {
+  const file = lockfilePath(shipRoot);
+  if (!fs.existsSync(file)) return null;
+  const raw = fs.readFileSync(file, "utf8");
+  let parsed;
+  try {
+    parsed = JSON.parse(raw);
+  } catch (err) {
+    throw new Error(`shipctl.lock.json: invalid JSON (${err instanceof Error ? err.message : err})`);
+  }
+  if (!parsed || typeof parsed !== "object") {
+    throw new Error("shipctl.lock.json: root must be an object");
+  }
+  if (parsed.version !== LOCKFILE_SCHEMA_VERSION) {
+    throw new Error(
+      `shipctl.lock.json: unsupported version ${parsed.version} (shipctl supports v${LOCKFILE_SCHEMA_VERSION}).`,
+    );
+  }
+  if (!parsed.artifacts || typeof parsed.artifacts !== "object") {
+    throw new Error("shipctl.lock.json: missing `artifacts` map");
+  }
+  return parsed;
+}
+
+/**
+ * @param {string} shipRoot
+ * @param {Lockfile} data
+ */
+export function writeLockfile(shipRoot, data) {
+  const file = lockfilePath(shipRoot);
+  fs.mkdirSync(path.dirname(file), { recursive: true });
+
+  /* Sort the artifacts map so diffs stay minimal between runs. We can't
+   * rely on JSON object key order, but `JSON.stringify` honours insertion
+   * order on v8, and every modern engine we care about follows suit. */
+  const sorted = {};
+  const keys = Object.keys(data.artifacts || {}).sort();
+  for (const k of keys) sorted[k] = data.artifacts[k];
+
+  const normalised = {
+    version: LOCKFILE_SCHEMA_VERSION,
+    generated_at: data.generated_at,
+    shipctl_version: data.shipctl_version,
+    source: data.source || null,
+    artifacts: sorted,
+    notes: Array.isArray(data.notes) ? [...data.notes] : [],
+  };
+
+  const tmp = `${file}.tmp`;
+  fs.writeFileSync(tmp, `${JSON.stringify(normalised, null, 2)}\n`, "utf8");
+  fs.renameSync(tmp, file);
+}
+
+/**
+ * Build the lookup key the rest of shipctl uses.
+ * @param {string} kind
+ * @param {string} id
+ */
+export function lockKey(kind, id) {
+  return `${kind}/${id}`;
+}
+
+/**
+ * @param {Lockfile} lock
+ * @param {string} kind
+ * @param {string} id
+ * @returns {LockfileEntry | null}
+ */
+export function lookupLock(lock, kind, id) {
+  if (!lock) return null;
+  const key = lockKey(kind, id);
+  const entry = lock.artifacts?.[key];
+  return entry || null;
+}
+
+/**
+ * Compute the lockfile entry for a freshly-materialised artifact body.
+ * Keeping the signature narrow — just `body` and a bit of provenance —
+ * means the caller can't accidentally smuggle transient fields into the
+ * lock.
+ *
+ * @param {Object} params
+ * @param {string} params.body           Artifact text (markdown + frontmatter).
+ * @param {string} params.version        Resolved version string.
+ * @param {string} params.cachedPath     Path relative to the ship root.
+ * @param {"http" | "monorepo" | "inline"} params.source
+ * @param {boolean} [params.pinned=false]
+ * @param {string} [params.channel]
+ * @returns {LockfileEntry}
+ */
+export function entryFromBody({
+  body,
+  version,
+  cachedPath,
+  source,
+  pinned = false,
+  channel,
+}) {
+  return {
+    version,
+    content_sha256: artifactSha256(body),
+    cached_path: String(cachedPath).replace(/\\/g, "/"),
+    source,
+    pinned: Boolean(pinned),
+    ...(channel ? { channel: String(channel) } : {}),
+  };
+}
+
+/**
+ * Decide whether a body matches a lockfile entry. Returns a structured
+ * `{ ok, reason }` so callers can emit specific diagnostics.
+ *
+ * @param {LockfileEntry | null} entry
+ * @param {string} body
+ * @returns {{ ok: boolean, reason?: string, expected?: string, actual?: string }}
+ */
+export function verifyBody(entry, body) {
+  if (!entry) return { ok: false, reason: "missing-entry" };
+  const actual = artifactSha256(body);
+  if (actual !== entry.content_sha256) {
+    return {
+      ok: false,
+      reason: "sha-mismatch",
+      expected: entry.content_sha256,
+      actual,
+    };
+  }
+  return { ok: true };
+}

package/lib/telemetry/outbox.mjs

@@ -0,0 +1,224 @@
+import fs from "node:fs";
+import path from "node:path";
+import { readConfig } from "../config/io.mjs";
+
+export const ALLOWED_EVENT_TYPES = Object.freeze([
+  "artifact.fetch",
+  "artifact.use",
+  "artifact.sync",
+  "feedback.submit",
+  "doctor.result",
+]);
+
+export const DENYLIST_KEYS = Object.freeze([
+  "path",
+  "code",
+  "diff",
+  "branch",
+  "remote",
+  "email",
+]);
+
+const ALLOWED_SET = new Set(ALLOWED_EVENT_TYPES);
+const DENY_SET = new Set(DENYLIST_KEYS.map((k) => k.toLowerCase()));
+
+export function outboxPath(shipRoot) {
+  return path.join(shipRoot, ".ship", "telemetry-outbox.jsonl");
+}
+
+function debug(msg) {
+  if (process.env.SHIP_DEBUG === "1") {
+    console.error(`[ship:telemetry] ${msg}`);
+  }
+}
+
+/**
+ * Recursively strip denylisted keys from a payload object, returning a new object.
+ * Returns {stripped, removed: string[]}.
+ */
+function scrubPayload(payload) {
+  const removed = [];
+  function walk(node) {
+    if (node === null || typeof node !== "object") return node;
+    if (Array.isArray(node)) return node.map(walk);
+    const out = {};
+    for (const [k, v] of Object.entries(node)) {
+      if (typeof k === "string" && DENY_SET.has(k.toLowerCase())) {
+        removed.push(k);
+        continue;
+      }
+      out[k] = walk(v);
+    }
+    return out;
+  }
+  const stripped = walk(payload ?? {});
+  return { stripped, removed };
+}
+
+/**
+ * Normalize an incoming event into the backend-compatible envelope:
+ *   { type, anonymous_id, timestamp, payload, shipctl_version?, stack_preset? }
+ *
+ * Accepts both the old shape ({event, ts}) emitted by earlier sync code and the
+ * new shape ({type, timestamp}). Only `type` is considered authoritative for
+ * whitelist validation.
+ */
+function normalizeEvent(input) {
+  if (!input || typeof input !== "object") {
+    throw new Error("appendEvent: event must be an object");
+  }
+  const type = input.type || input.event;
+  const timestamp = input.timestamp || input.ts || new Date().toISOString();
+  const payload = input.payload || {};
+  const out = {
+    type,
+    anonymous_id: input.anonymous_id,
+    timestamp,
+    payload,
+  };
+  if (input.shipctl_version) out.shipctl_version = input.shipctl_version;
+  if (input.stack_preset !== undefined) out.stack_preset = input.stack_preset;
+  return out;
+}
+
+function readConfigSafe(shipRoot) {
+  try {
+    const { config } = readConfig(shipRoot);
+    return config;
+  } catch {
+    return null;
+  }
+}
+
+/**
+ * Append one event to the outbox. Silently no-ops when telemetry is disabled.
+ * Throws on unknown event type. Strips denylisted keys from payload (quietly
+ * logging under SHIP_DEBUG=1).
+ *
+ * @param {string} shipRoot
+ * @param {object} event
+ * @returns {boolean} true if appended, false if skipped (telemetry off).
+ */
+export function appendEvent(shipRoot, event) {
+  const normalized = normalizeEvent(event);
+  if (!normalized.type || !ALLOWED_SET.has(normalized.type)) {
+    throw new Error(
+      `appendEvent: unknown event type ${JSON.stringify(normalized.type)}; allowed=${ALLOWED_EVENT_TYPES.join(",")}`,
+    );
+  }
+
+  const cfg = readConfigSafe(shipRoot);
+  if (!cfg || cfg.telemetry?.share !== true) return false;
+  if (String(process.env.SHIP_TELEMETRY || "").toLowerCase() === "false") return false;
+
+  if (!normalized.anonymous_id) {
+    normalized.anonymous_id = cfg.telemetry?.anonymous_id || null;
+  }
+  if (!normalized.anonymous_id) return false;
+
+  const { stripped, removed } = scrubPayload(normalized.payload);
+  if (removed.length) {
+    debug(`stripped denylisted keys from ${normalized.type}: ${removed.join(", ")}`);
+  }
+  normalized.payload = stripped;
+
+  const file = outboxPath(shipRoot);
+  fs.mkdirSync(path.dirname(file), { recursive: true });
+  fs.appendFileSync(file, `${JSON.stringify(normalized)}\n`, "utf8");
+  return true;
+}
+
+/**
+ * Read and parse events from the outbox. Lines that fail to parse are skipped
+ * with a debug warning. Old-shape events (`event`/`ts`) are upgraded on read.
+ */
+export function listEvents(shipRoot) {
+  const file = outboxPath(shipRoot);
+  if (!fs.existsSync(file)) return [];
+  const raw = fs.readFileSync(file, "utf8");
+  const out = [];
+  for (const line of raw.split(/\r?\n/)) {
+    const trimmed = line.trim();
+    if (!trimmed) continue;
+    let parsed;
+    try {
+      parsed = JSON.parse(trimmed);
+    } catch {
+      debug(`skipping malformed line in ${file}`);
+      continue;
+    }
+    const upgraded = {
+      type: parsed.type || parsed.event,
+      anonymous_id: parsed.anonymous_id,
+      timestamp: parsed.timestamp || parsed.ts,
+      payload: parsed.payload || {},
+    };
+    if (parsed.shipctl_version) upgraded.shipctl_version = parsed.shipctl_version;
+    if (parsed.stack_preset !== undefined) upgraded.stack_preset = parsed.stack_preset;
+    out.push(upgraded);
+  }
+  return out;
+}
+
+export function countEvents(shipRoot) {
+  return listEvents(shipRoot).length;
+}
+
+/**
+ * Remove events older than `before` (ISO string). If `before` is omitted,
+ * clears the entire outbox. Returns the number of events removed.
+ */
+export function clearEvents(shipRoot, { before } = {}) {
+  const file = outboxPath(shipRoot);
+  if (!fs.existsSync(file)) return 0;
+  if (!before) {
+    const before_count = countEvents(shipRoot);
+    try {
+      fs.unlinkSync(file);
+    } catch {
+      fs.writeFileSync(file, "", "utf8");
+    }
+    return before_count;
+  }
+
+  const beforeMs = Date.parse(before);
+  if (Number.isNaN(beforeMs)) {
+    throw new Error(`clearEvents: invalid 'before' timestamp ${JSON.stringify(before)}`);
+  }
+  const kept = [];
+  let removed = 0;
+  for (const ev of listEvents(shipRoot)) {
+    const ts = Date.parse(ev.timestamp || "");
+    if (!Number.isNaN(ts) && ts < beforeMs) {
+      removed += 1;
+    } else {
+      kept.push(ev);
+    }
+  }
+  if (kept.length === 0) {
+    try {
+      fs.unlinkSync(file);
+    } catch {
+      fs.writeFileSync(file, "", "utf8");
+    }
+  } else {
+    const text = kept.map((e) => JSON.stringify(e)).join("\n") + "\n";
+    fs.writeFileSync(file, text, "utf8");
+  }
+  return removed;
+}
+
+/**
+ * Overwrite the outbox with the given list of event envelopes. Used by flush
+ * to persist the subset of lines that failed to POST.
+ */
+export function writeAllEvents(shipRoot, events) {
+  const file = outboxPath(shipRoot);
+  if (!events.length) {
+    if (fs.existsSync(file)) fs.unlinkSync(file);
+    return;
+  }
+  fs.mkdirSync(path.dirname(file), { recursive: true });
+  const text = events.map((e) => JSON.stringify(e)).join("\n") + "\n";
+  fs.writeFileSync(file, text, "utf8");
+}