@indigoai-us/hq-cloud 6.14.30 → 6.14.32

package/src/qmd-reindex.ts

@@ -15,9 +15,10 @@
  * script inside the synced HQ tree (which may be stale).
  *
  * What it does, best-effort and idempotent:
- *   1. Auto-registers any `companies/<slug>/knowledge` dir that isn't yet a
- *      qmd collection (kills the manual "map" step). Detects path drift when
- *      the name exists but points elsewhere; repairs only when
+ *   1. Auto-registers populated company knowledge, company project, and
+ *      personal knowledge dirs that aren't yet qmd collections (kills the
+ *      manual "map" step). Company knowledge detects path drift when the name
+ *      exists but points elsewhere; repairs only when
  *      `HQ_QMD_REPAIR_PATH_DRIFT=1` (non-breaking default: detect-only).
  *   2. Runs an incremental lexical `qmd update` (fast — qmd skips unchanged
  *      files by mtime).
@@ -395,6 +396,8 @@ export interface ReindexOptions {
   readCompanies?: (companiesDir: string) => string[];
   /** Returns true if the knowledge dir has at least one indexable .md file. */
   hasIndexableMarkdown?: (knowledgeDir: string) => boolean;
+  /** Returns true if the projects dir has at least one indexable .md or .json file. */
+  hasIndexableProjectContent?: (projectsDir: string) => boolean;
   /** Reindex-lock acquirer override for tests. */
   acquireReindexLock?: AcquireReindexLock;
   /** Corrupt-index quarantine override for tests. */
@@ -474,7 +477,7 @@ export function reindexAfterSync(
     const changedPaths = opts.changedPaths;
     const dirtyFromChanges =
       changedPaths === undefined ||
-      changedPaths.some(isKnowledgeMarkdownPath);
+      changedPaths.some(isQmdIndexableContentPath);
     const registrationMayBeStale =
       opts.forceCollectionRefresh === true ||
       changedPaths === undefined ||
@@ -528,6 +531,9 @@ export function reindexAfterSync(
       result.qmdAvailable = true;
       const existingCollections = list.stdout;
       let repairedThisCycle = false;
+      const hasIndexableMarkdown = opts.hasIndexableMarkdown ?? defaultHasIndexableMarkdown;
+      const hasIndexableProjectContent =
+        opts.hasIndexableProjectContent ?? defaultHasIndexableProjectContent;
 
       // 1. Auto-register missing / reconcile drifted company knowledge collections.
       if (registrationMayBeStale) {
@@ -537,7 +543,7 @@ export function reindexAfterSync(
         for (const slug of slugs) {
           const knowledgeDir = path.join(companiesDir, slug, "knowledge");
           if (!existsSync(knowledgeDir)) continue;
-          const hasMd = (opts.hasIndexableMarkdown ?? defaultHasIndexableMarkdown)(knowledgeDir);
+          const hasMd = hasIndexableMarkdown(knowledgeDir);
           if (!hasMd) continue;
 
           const namePresent = existingCollections.includes(`qmd://${slug}/`);
@@ -601,6 +607,58 @@ export function reindexAfterSync(
           result.collectionsRepaired.push(slug);
           repairedThisCycle = true;
         }
+
+        // 1b. Auto-register company projects collections. Match the shell
+        // post-sync convention exactly so prd.json and project docs are searchable.
+        for (const slug of slugs) {
+          const projectsDir = path.join(companiesDir, slug, "projects");
+          if (!existsSync(projectsDir)) continue;
+          if (!hasIndexableProjectContent(projectsDir)) continue;
+
+          const name = `${slug}-projects`;
+          if (existingCollections.includes(`qmd://${name}/`)) continue;
+          const add = exec([
+            "collection",
+            "add",
+            projectsDir,
+            "--name",
+            name,
+            "--mask",
+            "**/*.{md,json}",
+          ]);
+          if (add.status === 0) {
+            exec(["context", "add", `qmd://${name}`, `Project PRDs and documentation for ${slug}.`]);
+            result.collectionsAdded.push(name);
+          }
+        }
+
+        // 1c. Auto-register the owner overlay's personal knowledge collection.
+        const personalKnowledgeDir = path.join(hqRoot, "personal", "knowledge");
+        const personalKnowledgeName = "personal-knowledge";
+        if (
+          existsSync(personalKnowledgeDir) &&
+          hasIndexableMarkdown(personalKnowledgeDir) &&
+          !existingCollections.includes(`qmd://${personalKnowledgeName}/`)
+        ) {
+          const add = exec([
+            "collection",
+            "add",
+            personalKnowledgeDir,
+            "--name",
+            personalKnowledgeName,
+            "--mask",
+            "**/*.md",
+          ]);
+          if (add.status === 0) {
+            exec([
+              "context",
+              "add",
+              `qmd://${personalKnowledgeName}`,
+              "Personal knowledge base (owner overlay).",
+            ]);
+            result.collectionsAdded.push(personalKnowledgeName);
+          }
+        }
       }
 
       // 2. Incremental lexical reindex.
@@ -721,9 +779,13 @@ function writeState(statePath: string, state: QmdReindexState): void {
   }
 }
 
-function isKnowledgeMarkdownPath(relPath: string): boolean {
+function isQmdIndexableContentPath(relPath: string): boolean {
   const normalized = relPath.split(path.sep).join("/");
-  return /^companies\/[^/]+\/knowledge\/.+\.md$/i.test(normalized);
+  return (
+    /^companies\/[^/]+\/knowledge\/.+\.md$/i.test(normalized) ||
+    /^companies\/[^/]+\/projects\/.+\.(?:md|json)$/i.test(normalized) ||
+    /^personal\/knowledge\/.+\.md$/i.test(normalized)
+  );
 }
 
 function defaultReadCompanies(companiesDir: string): string[] {
@@ -758,3 +820,23 @@ function defaultHasIndexableMarkdown(knowledgeDir: string): boolean {
   }
   return false;
 }
+
+function defaultHasIndexableProjectContent(projectsDir: string): boolean {
+  // Match the shell's `find -type f \( -name '*.md' -o -name '*.json' \)`
+  // guard: project documentation and PRD JSON both make the collection useful.
+  const stack = [projectsDir];
+  while (stack.length) {
+    const dir = stack.pop()!;
+    let entries: fs.Dirent[];
+    try {
+      entries = fs.readdirSync(dir, { withFileTypes: true });
+    } catch {
+      continue;
+    }
+    for (const e of entries) {
+      if (e.isFile() && (e.name.endsWith(".md") || e.name.endsWith(".json"))) return true;
+      if (e.isDirectory()) stack.push(path.join(dir, e.name));
+    }
+  }
+  return false;
+}

package/src/telemetry.test.ts

@@ -126,6 +126,8 @@ function makeOpts(env: TestEnv, client: TelemetryClientSurface): CollectTelemetr
     installerVersion: "test-version",
     claudeProjectsRoot: env.claudeProjects,
     codexRoot: env.codexRoot,
+    maxScanBytesPerSource: Number.MAX_SAFE_INTEGER,
+    maxBatchesPerRun: Number.MAX_SAFE_INTEGER,
     cursorPath: env.cursorPath,
     menubarPath: env.menubarPath,
   };
@@ -813,6 +815,96 @@ describe("collectAndSendTelemetry — Codex rollouts", () => {
     expect(cursor.codex_next_rollout).toBe(file);
   });
 
+  it("bounds a quiet first backfill and resumes until the token event", async () => {
+    const file = writeCodexRollout(env, "sessions", "rollout-bounded.jsonl", [
+      { type: "session_meta", payload: { id: "bounded", model: "gpt-5-codex" } },
+      ...Array.from({ length: 8 }, (_, index) => ({
+        type: "response_item",
+        payload: { type: "message", content: "x".repeat(140), index },
+      })),
+      codexToken("2026-07-30T14:00:00Z", "bounded-event", 5, 7),
+    ]);
+    const client = makeClient();
+    const opts = {
+      ...makeOpts(env, client),
+      maxScanBytesPerSource: 350,
+    };
+
+    const first = await collectAndSendTelemetry(opts);
+
+    expect(first.eventsSent).toBe(0);
+    const firstOffset = readCursor(env).files[file]?.offset ?? 0;
+    expect(firstOffset).toBeGreaterThan(0);
+    expect(firstOffset).toBeLessThan(fs.statSync(file).size);
+
+    for (let run = 0; run < 10 && client.posts.length === 0; run++) {
+      await collectAndSendTelemetry(opts);
+    }
+    expect(client.posts.flatMap((post) => post.events)).toEqual([
+      expect.objectContaining({
+        sessionId: "bounded",
+        uuid: "bounded-event",
+        model: "gpt-5-codex",
+      }),
+    ]);
+  });
+
+  it("discards an oversized UTF-8 row without corrupting resume offsets", async () => {
+    const file = writeCodexRollout(env, "sessions", "rollout-utf8.jsonl", [
+      { type: "session_meta", payload: { id: "utf8", model: "gpt-5-codex" } },
+      {
+        type: "response_item",
+        payload: { type: "message", content: "😀".repeat(20_000) },
+      },
+      codexToken("2026-07-30T14:00:01Z", "utf8-event", 2, 3),
+    ]);
+    const client = makeClient();
+    const opts = {
+      ...makeOpts(env, client),
+      maxScanBytesPerSource: 64 * 1024,
+    };
+
+    for (let run = 0; run < 5 && client.posts.length === 0; run++) {
+      await collectAndSendTelemetry(opts);
+    }
+
+    const events = client.posts.flatMap((post) => post.events);
+    expect(events).toEqual([
+      expect.objectContaining({ uuid: "utf8-event", sessionId: "utf8" }),
+    ]);
+    expect(readCursor(env).files[file].offset).toBeLessThanOrEqual(
+      fs.statSync(file).size,
+    );
+  });
+
+  it("caps upload batches and resumes remaining token events", async () => {
+    writeCodexRollout(env, "sessions", "rollout-batch-cap.jsonl", [
+      { type: "session_meta", payload: { id: "batch-cap", model: "gpt-5-codex" } },
+      ...Array.from({ length: 201 }, (_, index) =>
+        codexToken(
+          "2026-07-30T14:01:" + String(index % 60).padStart(2, "0") + "Z",
+          "batch-cap-" + index,
+          1,
+          1,
+        ),
+      ),
+    ]);
+    const client = makeClient();
+    const opts = {
+      ...makeOpts(env, client),
+      maxScanBytesPerSource: 1_000_000,
+      maxBatchesPerRun: 2,
+    };
+
+    const first = await collectAndSendTelemetry(opts);
+    expect(first.batchesSent).toBe(2);
+    expect(first.eventsSent).toBe(200);
+
+    const second = await collectAndSendTelemetry(opts);
+    expect(second.eventsSent).toBe(1);
+    expect(client.posts.flatMap((post) => post.events)).toHaveLength(201);
+  });
+
   it("does not advance a Codex rollout cursor when upload fails", async () => {
     const file = writeCodexRollout(env, "sessions", "rollout-retry.jsonl", [
       { type: "session_meta", payload: { id: "retry", model: "gpt-5-codex" } },
@@ -886,6 +978,35 @@ describe("collectAndSendTelemetry — companyUid attribution", () => {
     expect("companyUid" in client.posts[0].events[0]).toBe(false);
   });
 
+  it("uses the explicit single-company fallback when cwd is the shared HQ root", async () => {
+    const client = makeClient();
+    writeJsonl(env, "proj", "s.jsonl", [rowWithCwd("u1", hqRoot)]);
+
+    await collectAndSendTelemetry({
+      ...makeOpts(env, client),
+      hqRoot,
+      fallbackCompany: "indigo",
+    });
+
+    expect(client.posts).toHaveLength(1);
+    expect(client.posts[0].events[0].companyUid).toBe(COMPANY_UID);
+  });
+
+  it("prefers the cwd-resolved company over the single-company fallback", async () => {
+    const client = makeClient();
+    const inRepo = path.join(hqRoot, "repos/private/hq-cloud");
+    writeJsonl(env, "proj", "s.jsonl", [rowWithCwd("u1", inRepo)]);
+
+    await collectAndSendTelemetry({
+      ...makeOpts(env, client),
+      hqRoot,
+      fallbackCompany: "cmp_01OTHER",
+    });
+
+    expect(client.posts).toHaveLength(1);
+    expect(client.posts[0].events[0].companyUid).toBe(COMPANY_UID);
+  });
+
   it("never sends the reserved 'unattributed' value", async () => {
     const client = makeClient();
     writeJsonl(env, "proj", "s.jsonl", [