@indigoai-us/hq-cli 5.89.0 → 5.89.2

package/CHANGELOG.md

@@ -2,6 +2,31 @@
 
 ## [Unreleased]
 
+## [5.89.2]
+
+### Fixed
+
+- The `hq core checkpoint` maintenance sibling no longer loses queued payloads.
+  Both drain paths read `pending.jsonl` and then truncated it, so a checkpoint
+  queued between the read and the truncate was discarded unread and never
+  enriched. The queue is now claimed atomically by renaming it into the run
+  directory, leaving producers a fresh queue and keeping the claimed file as an
+  audit and crash-recovery trail; the sibling prompt claims by `mv` in a loop
+  rather than truncating in place. (#317)
+
+## [5.89.1]
+
+### Fixed
+
+- `hq core worker lint` now partitions its duplicate check by ownership domain
+  (`core/workers`, each `core/packages/<pack>`, each `companies/<co>`) and
+  excludes dot-directories (`.archive/`) and `._*` AppleDouble files. A
+  whole-tree scan of a real install previously reported false positives for
+  packs that intentionally vendor a byte-identical copy of a core worker skill,
+  for archived pack snapshots, and for macOS metadata files. Within-domain
+  duplicates are still flagged. (policy
+  hq-lint-comparison-domains-must-match-ownership)
+
 ## [5.89.0]
 
 ### Added

package/assets/scaffold/core/scripts/lint-shared-worker-skills.sh

@@ -12,22 +12,37 @@
 #
 # This linter makes that convention enforceable. It flags two failure shapes:
 #
-#   DUPLICATE  — two or more NON-symlink skill files, in any worker under the
-#                scanned roots, whose byte content is identical. Identical bytes
-#                across two real files is exactly an un-single-sourced copy: the
-#                moment someone edits one, they drift. (Symlinks that resolve to
-#                a shared canonical are single-sourced and are NOT flagged, even
-#                though their resolved content matches the canonical.)
+#   DUPLICATE  — two or more NON-symlink skill files, WITHIN A SINGLE OWNERSHIP
+#                DOMAIN, whose byte content is identical. Identical bytes across
+#                two real files in the same domain is exactly an un-single-sourced
+#                copy: the moment someone edits one, they drift. (Symlinks that
+#                resolve to a shared canonical are single-sourced and are NOT
+#                flagged, even though their resolved content matches.)
 #   BROKEN     — a skill entry that is a symlink whose target does not resolve.
 #
 # It deliberately does NOT flag two same-NAMED skills whose content differs
 # (e.g. an API-level vs a browser-level e2e skill): distinct content means they
 # are distinct skills that merely share a filename, not a drifted share.
 #
+# OWNERSHIP DOMAINS — filesystem adjacency does NOT establish duplicate
+# ownership, so comparison is partitioned by the tree that OWNS a skill (policy
+# hq-lint-comparison-domains-must-match-ownership):
+#
+#   - `core/workers`            — the curated first-party worker library
+#   - `core/packages/<pack>`    — one installed pack (self-contained; may legally
+#                                 vendor a byte-identical copy of a core worker)
+#   - `companies/<co>`          — one tenant's workers
+#
+# A byte-identical file in TWO DIFFERENT domains (a pack vendoring a core skill,
+# or two packs shipping the same skill) is intentional distribution, NOT drift,
+# and is never flagged. Excluded from the scan entirely: dot-directories
+# (`.archive/`, `.git/`, …) and `._*` AppleDouble OS-metadata files — byte-
+# identical by design, and not skills.
+#
 # Usage: lint-shared-worker-skills.sh [root ...]
 #   Default roots: core/workers core/packages  (the SHIPPED hq-core scope —
-#   company `_shared-skills/` stores live under companies/<co>/ and are linted
-#   per-tenant, not here).
+#   company workers live under companies/<co>/ and are linted per-tenant; pass
+#   them explicitly to include them).
 #
 # Exit 0 + "OK:" line when clean; exit 1 + a report naming every offending group
 # when not. Matches the loud-and-specific style of lint-skill-script-refs.sh.
@@ -54,11 +69,29 @@ content_hash() {
   fi
 }
 
+# The ownership domain that OWNS a skill path — the boundary within which two
+# byte-identical files are a real duplicate. Across domains, identical content
+# is intentional (a pack vendoring a core skill, two packs shipping the same
+# skill), so it must never be compared. See policy header above.
+domain_of() {
+  local p="${1#./}" rest
+  case "$p" in
+    core/packages/*) rest="${p#core/packages/}"; printf 'core/packages/%s' "${rest%%/*}" ;;
+    companies/*)     rest="${p#companies/}";      printf 'companies/%s' "${rest%%/*}" ;;
+    core/workers/*|core/workers) printf 'core/workers' ;;
+    # Arbitrary root (custom scans, tests): the top path segment is the domain.
+    *) printf '%s' "${p%%/*}" ;;
+  esac
+}
+
 # Collect skill entries: regular files AND symlinks named *.md under any
 # .../skills/... path. -type l must be matched explicitly — a symlink is not a
 # -type f, so a skills symlink would otherwise be invisible to the scan.
+# EXCLUDED: any dot-directory component (`*/.*/*` — covers `.archive/`, `.git/`)
+# and `._*` AppleDouble files; both are byte-identical by design and not skills.
 mapfile -t entries < <(
-  find "${roots[@]}" \( -type f -o -type l \) -path '*/skills/*.md' 2>/dev/null | sort
+  find "${roots[@]}" \( -type f -o -type l \) -path '*/skills/*.md' \
+    -not -path '*/.*/*' -not -name '._*' 2>/dev/null | sort
 )
 
 # Empty is clean — and guards `"${entries[@]}"` under `set -u` on bash 3.2
@@ -69,10 +102,13 @@ if [[ ${#entries[@]} -eq 0 ]]; then
 fi
 
 broken=()
-# Parallel arrays keyed by content hash: hash_keys[i] is a hash, and
-# hash_regfiles[i] is a newline-joined list of the NON-symlink files with that
-# hash. Bash 3.2 (macOS) has no associative arrays in a portable-guaranteed way,
-# so a linear scan over parallel arrays keeps this runnable everywhere HQ runs.
+# Parallel arrays keyed by "<domain>|<content-hash>": hash_keys[i] is that
+# composite key, hash_regfiles[i] a newline-joined list of the NON-symlink files
+# carrying it. Keying on domain AS WELL AS hash is what confines a duplicate
+# finding to a single ownership tree — two identical files in different domains
+# land in different buckets and are never flagged. Bash 3.2 (macOS) has no
+# portable associative arrays, so a linear scan over parallel arrays keeps this
+# runnable everywhere HQ runs.
 hash_keys=()
 hash_regfiles=()
 
@@ -94,12 +130,13 @@ for entry in "${entries[@]}"; do
     fi
     continue
   fi
-  # Regular file: hash its bytes and bucket it. Two regular files sharing a
-  # bucket are two copies of the same skill — the drift hazard.
-  h="$(content_hash "$entry")"
-  idx="$(hash_index "$h")"
+  # Regular file: bucket it by (ownership domain, byte content). Two regular
+  # files sharing a bucket are two copies of the same skill in the same tree —
+  # the drift hazard.
+  key="$(domain_of "$entry")|$(content_hash "$entry")"
+  idx="$(hash_index "$key")"
   if [[ "$idx" == "-1" ]]; then
-    hash_keys+=("$h")
+    hash_keys+=("$key")
     hash_regfiles+=("$entry")
   else
     hash_regfiles[$idx]="${hash_regfiles[$idx]}"$'\n'"$entry"
@@ -116,7 +153,8 @@ for i in "${!hash_keys[@]}"; do
       echo "lint-shared-worker-skills: FAIL — duplicated worker skills (single-source these via _shared-skills/ + relative symlinks):" >&2
     fi
     findings=$((findings + 1))
-    echo "  DUPLICATE (identical content, ${count} copies — pick one canonical and symlink the rest):" >&2
+    domain="${hash_keys[$i]%%|*}"
+    echo "  DUPLICATE in domain '${domain}' (identical content, ${count} copies — pick one canonical and symlink the rest):" >&2
     printf '    %s\n' "$group" >&2
   fi
 done

package/dist/commands/core-checkpoint.d.ts

@@ -11,7 +11,7 @@ import { Command } from "commander";
  * Kept in TypeScript rather than in a bundled asset: it is an instruction to
  * a locally-installed agent, not a scaffold script that should be packaged.
  */
-export declare const SIBLING_PROMPT_TEMPLATE = "You are the HQ checkpoint sibling \u2014 a background maintenance agent for this\nHQ install. Your parent session's state is in <payloadPath>. Work\nquietly and do not ask questions; if something is ambiguous, record it in the\nreport instead of guessing.\n\n1. Read the payload. If it lists a transcript path that exists, read its tail (~400 lines)\n   both for session context and to extract additional reusable learnings/insights\n   the parent did not pass explicitly. Never quote secrets\n   or tokens from the transcript. If .claude/skills/checkpoint/SKILL.md exists\n   under this HQ root, read it and follow it wherever it goes beyond these instructions;\n   the write bounds below always win over the skill text.\n2. Upgrade the thread file named in the payload IN PLACE: verify/repair its\n   JSON; fill git.remote_url, git.initial_commit, git.commits_made, and\n   git.knowledge_repos by scanning core/knowledge/public/*,\n   core/knowledge/private/*, personal/knowledge/*, and companies/*/knowledge\n   for symlinks or directories containing .git, recording dirty repositories\n   as {\"<name>\": {\"commit\": \"<short>\", \"dirty\": true}}. Fill worker,\n   next_steps, and insights; set type to \"checkpoint\"; then rename the file to\n   drop -auto- from its filename. Use the renamed path in every reference you\n   write afterwards.\n3. For every explicit or transcript-derived learning that is a reusable rule,\n   distill a non-duplicate policy file under personal/policies/ or, only when\n   the payload names a company and the rule is company-specific,\n   companies/<company>/policies/, following\n   core/knowledge/public/hq-core/policies-spec.md. Store up to two explicit or\n   transcript-derived insights per core/knowledge/public/hq-core/insights-spec.md\n   when present, otherwise workspace/insights/. Durable facts (not rules) may\n   go under personal/knowledge/ or companies/<company>/knowledge/ only.\n4. Close an active session journal fail-soft with\n   bash .claude/skills/_shared/journal.sh close \"<project_dir>\" \"<one-line synthesis>\".\n   Write a legacy checkpoint JSON under workspace/checkpoints/<id>.json with\n   id, created_at, summary, files, and next_steps for backward compatibility.\n5. Update workspace/threads/recent.md and regenerate\n   workspace/threads/INDEX.md. For each company whose knowledge path appears\n   in files_touched, regenerate companies/<company>/knowledge/INDEX.md under\n   core/knowledge/public/hq-core/index-md-spec.md. Mechanical index generation\n   is allowed for those companies, but knowledge/policy content writes remain\n   restricted to the payload's named company.\n6. Run .claude/skills/document-release/SKILL.md best-effort when it exists;\n   skip silently on any failure. Hook or automation improvements go ONLY under\n   personal/hooks/ as proposals.\n7. WRITE BOUNDS: you may write only under personal/, workspace/, and companies/<company>/ as constrained above. You must NEVER write into .claude/, core/, .agents/, .codex/, repos/, or anywhere outside the HQ root.\n8. Write <runDir>/report.md \u2014 full prose: what you read, what you changed\n   (paths), and what you skipped and why. If\n   workspace/checkpoints/sibling/pending.jsonl is non-empty when you finish,\n   process those payloads with this same flow, then truncate the file.\n";
+export declare const SIBLING_PROMPT_TEMPLATE = "You are the HQ checkpoint sibling \u2014 a background maintenance agent for this\nHQ install. Your parent session's state is in <payloadPath>. Work\nquietly and do not ask questions; if something is ambiguous, record it in the\nreport instead of guessing.\n\n1. Read the payload. If it lists a transcript path that exists, read its tail (~400 lines)\n   both for session context and to extract additional reusable learnings/insights\n   the parent did not pass explicitly. Never quote secrets\n   or tokens from the transcript. If .claude/skills/checkpoint/SKILL.md exists\n   under this HQ root, read it and follow it wherever it goes beyond these instructions;\n   the write bounds below always win over the skill text.\n2. Upgrade the thread file named in the payload IN PLACE: verify/repair its\n   JSON; fill git.remote_url, git.initial_commit, git.commits_made, and\n   git.knowledge_repos by scanning core/knowledge/public/*,\n   core/knowledge/private/*, personal/knowledge/*, and companies/*/knowledge\n   for symlinks or directories containing .git, recording dirty repositories\n   as {\"<name>\": {\"commit\": \"<short>\", \"dirty\": true}}. Fill worker,\n   next_steps, and insights; set type to \"checkpoint\"; then rename the file to\n   drop -auto- from its filename. Use the renamed path in every reference you\n   write afterwards.\n3. For every explicit or transcript-derived learning that is a reusable rule,\n   distill a non-duplicate policy file under personal/policies/ or, only when\n   the payload names a company and the rule is company-specific,\n   companies/<company>/policies/, following\n   core/knowledge/public/hq-core/policies-spec.md. Store up to two explicit or\n   transcript-derived insights per core/knowledge/public/hq-core/insights-spec.md\n   when present, otherwise workspace/insights/. Durable facts (not rules) may\n   go under personal/knowledge/ or companies/<company>/knowledge/ only.\n4. Close an active session journal fail-soft with\n   bash .claude/skills/_shared/journal.sh close \"<project_dir>\" \"<one-line synthesis>\".\n   Write a legacy checkpoint JSON under workspace/checkpoints/<id>.json with\n   id, created_at, summary, files, and next_steps for backward compatibility.\n5. Update workspace/threads/recent.md and regenerate\n   workspace/threads/INDEX.md. For each company whose knowledge path appears\n   in files_touched, regenerate companies/<company>/knowledge/INDEX.md under\n   core/knowledge/public/hq-core/index-md-spec.md. Mechanical index generation\n   is allowed for those companies, but knowledge/policy content writes remain\n   restricted to the payload's named company.\n6. Run .claude/skills/document-release/SKILL.md best-effort when it exists;\n   skip silently on any failure. Hook or automation improvements go ONLY under\n   personal/hooks/ as proposals.\n7. WRITE BOUNDS: you may write only under personal/, workspace/, and companies/<company>/ as constrained above. You must NEVER write into .claude/, core/, .agents/, .codex/, repos/, or anywhere outside the HQ root.\n8. Write <runDir>/report.md \u2014 full prose: what you read, what you changed\n   (paths), and what you skipped and why. Then drain the queue: while\n   workspace/checkpoints/sibling/pending.jsonl exists and is non-empty, claim\n   it atomically by renaming it aside \u2014\n   mv workspace/checkpoints/sibling/pending.jsonl <runDir>/pending-claimed-N.jsonl\n   (N counting up from 2) \u2014 and process the claimed payloads with this same\n   flow. Repeat until a claim finds nothing left, then update the report.\n   NEVER read the queue and truncate it in place: a payload appended between\n   your read and the truncate is lost, and its checkpoint is never enriched.\n";
 export declare function renderSiblingPrompt(runDir: string, payloadPath: string): string;
 /** Attach the native checkpoint command to the hidden `hq core` group. */
 export declare function registerCoreCheckpointCommand(core: Command): void;

package/dist/commands/core-checkpoint.js

@@ -78,9 +78,14 @@ report instead of guessing.
    personal/hooks/ as proposals.
 7. WRITE BOUNDS: you may write only under personal/, workspace/, and companies/<company>/ as constrained above. You must NEVER write into .claude/, core/, .agents/, .codex/, repos/, or anywhere outside the HQ root.
 8. Write <runDir>/report.md — full prose: what you read, what you changed
-   (paths), and what you skipped and why. If
-   workspace/checkpoints/sibling/pending.jsonl is non-empty when you finish,
-   process those payloads with this same flow, then truncate the file.
+   (paths), and what you skipped and why. Then drain the queue: while
+   workspace/checkpoints/sibling/pending.jsonl exists and is non-empty, claim
+   it atomically by renaming it aside —
+   mv workspace/checkpoints/sibling/pending.jsonl <runDir>/pending-claimed-N.jsonl
+   (N counting up from 2) — and process the claimed payloads with this same
+   flow. Repeat until a claim finds nothing left, then update the report.
+   NEVER read the queue and truncate it in place: a payload appended between
+   your read and the truncate is lost, and its checkpoint is never enriched.
 `;
 export function renderSiblingPrompt(runDir, payloadPath) {
     return SIBLING_PROMPT_TEMPLATE
@@ -457,26 +462,43 @@ function appendPending(pendingPath, payload) {
         .filter((line) => line.trim());
     if (lines.length <= MAX_PENDING_PAYLOADS)
         return lines.length;
+    // Best-effort trim: this rewrite can race a concurrent append, but it only
+    // runs once the queue is already over cap, where newest-wins is the point.
     const kept = lines.slice(-MAX_PENDING_PAYLOADS);
     fs.writeFileSync(pendingPath, `${kept.join("\n")}\n`);
     return kept.length;
 }
-function drainPending(pendingPath) {
-    if (!fs.existsSync(pendingPath))
-        return [];
+/**
+ * Take ownership of the queued payloads by renaming the queue aside.
+ *
+ * The queue must never be claimed by read-then-truncate: a payload appended
+ * between the read and the truncate is discarded unread, and the checkpoint it
+ * describes is never enriched. Renaming is atomic, so this run receives exactly
+ * the file that existed at the moment of the claim while concurrent callers go
+ * on appending to a fresh queue. The claimed file is left in the run directory
+ * as an audit trail and as a recovery point if this sibling dies.
+ */
+function claimPending(pendingPath, claimPath) {
+    try {
+        fs.renameSync(pendingPath, claimPath);
+    }
+    catch (error) {
+        if (error?.code === "ENOENT")
+            return [];
+        throw error;
+    }
     const entries = [];
-    for (const line of fs.readFileSync(pendingPath, "utf8").split("\n")) {
+    for (const line of fs.readFileSync(claimPath, "utf8").split("\n")) {
         if (!line.trim())
             continue;
         try {
             entries.push(JSON.parse(line));
         }
         catch {
-            // A partial line must not prevent a future sibling from processing the
-            // valid queued payloads around it.
+            // A partial line must not prevent this run from processing the valid
+            // queued payloads around it.
         }
     }
-    fs.writeFileSync(pendingPath, "");
     return entries;
 }
 function startSibling(liveRoot, input, threadPath, backend) {
@@ -494,7 +516,7 @@ function startSibling(liveRoot, input, threadPath, backend) {
     }
     const runDir = path.join(siblingRoot, `${formatTimestamp(new Date())}-${summarySlug(input.summary ?? "checkpoint")}`);
     fs.mkdirSync(runDir, { recursive: true });
-    payload.pending_payloads = drainPending(pendingPath);
+    payload.pending_payloads = claimPending(pendingPath, path.join(runDir, "pending-claimed.jsonl"));
     const payloadPath = path.join(runDir, "payload.json");
     fs.writeFileSync(payloadPath, `${JSON.stringify(payload, null, 2)}\n`);
     const prompt = renderSiblingPrompt(runDir, payloadPath);

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@indigoai-us/hq-cli",
-  "version": "5.89.0",
+  "version": "5.89.2",
   "description": "HQ by Indigo management CLI — modules and cloud sync",
   "main": "dist/index.js",
   "bin": {