@groundtrack/hooks 0.1.25 → 0.1.27

package/README.md

@@ -169,6 +169,31 @@ high-confidence extracted lessons—not transcript content—to the organization
 Changing transcript visibility does not change this learning behavior; deleting an imported
 session retracts its derived knowledge.
 
+## Watch completed sessions
+
+`groundtrack watch --once` is the sync engine behind a future scheduled watcher.
+One pass scans session file metadata, compares it with a durable cursor, and only
+reads and uploads files whose size or modification time changed. Files modified in
+the last five minutes stay `active`; once they are quiet, the pass prunes and
+uploads them using the same privacy filter and receipts as `import`.
+
+```sh
+GROUNDTRACK_API_KEY=... groundtrack watch --once --agent codex --dry-run
+GROUNDTRACK_API_KEY=... groundtrack watch --once --agent codex --all-repos --yes
+GROUNDTRACK_API_KEY=... groundtrack watch --once --agent claude --all-repos --yes
+```
+
+The cursor lives at `$XDG_CONFIG_HOME/groundtrack/watch-state-<source>.json` (mode
+`0600`). Unchanged files are not reopened, rehashed, or re-uploaded. A preview
+without `--yes` does not write cursor state. Only one watch pass can run per source
+at a time; a stale PID lock is recovered automatically. The loop, filesystem
+events, and an OS timer are intentionally not part of `--once`.
+
+Codex paginated sessions are merged before upload. The watcher groups rollout
+pages by `session_id`, follows the `history_base.thread_id` chain, and uploads one
+complete snapshot. A chain with a missing or still-live page is reported instead
+of uploading a fragment.
+
 `templates/AGENTS.md` is the source of truth for the managed repository policy.
 After changing it, run `pnpm sync:agent-instructions`; tests reject drift between
 the setup guide, this repository's managed `AGENTS.md` block, and the packaged template.

package/dist/groundtrack.js

@@ -1,8 +1,13 @@
 #!/usr/bin/env node
 import { runCli } from "./setup/cli.js";
 import { runImportCli } from "./session-import/cli.js";
+import { runWatchCli } from "./session-import/watch.js";
 const argv = process.argv.slice(2);
-const operation = argv[0] === "import" ? runImportCli(argv.slice(1)) : runCli(argv);
+const operation = argv[0] === "import"
+    ? runImportCli(argv.slice(1))
+    : argv[0] === "watch"
+        ? runWatchCli(argv.slice(1))
+        : runCli(argv);
 operation.catch((error) => {
     const message = error instanceof Error ? error.message : "Groundtrack command failed.";
     if (process.argv.includes("--json"))

package/dist/session-import/canonical.js

@@ -1,5 +1,5 @@
 export const CANONICALIZATION_VERSION = 2;
-export const CANONICAL_IMPORT_MAX_BYTES = 50 * 1024 * 1024;
+export const CANONICAL_IMPORT_MAX_BYTES = 100 * 1024 * 1024;
 export const CANONICAL_IMPORT_MAX_RECORD_BYTES = 4 * 1024 * 1024;
 export const CANONICAL_IMPORT_MAX_ENTRIES = 25_000;
 export function isSessionRecord(value) {

package/dist/session-import/cli.d.ts

@@ -1,6 +1,6 @@
 import type { ImportAgent, ImportVisibility } from "./types.js";
 export declare const IMPORT_USAGE: string;
-type ImportArguments = Readonly<{
+export type ImportArguments = Readonly<{
     agent: ImportAgent | null;
     repoDir?: string;
     homeDir?: string;
@@ -20,4 +20,3 @@ type ImportArguments = Readonly<{
 export declare function normalizeApiBaseUrl(value: string): string;
 export declare function parseImportArguments(argv: readonly string[]): ImportArguments;
 export declare function runImportCli(argv: readonly string[]): Promise<void>;
-export {};

package/dist/session-import/cli.js

@@ -2,12 +2,11 @@ import { basename, resolve } from "node:path";
 import * as prompts from "@clack/prompts";
 import { isSessionImportAgent, SESSION_IMPORT_AGENTS } from "../agents.js";
 import { discoverAgents, environment, savedAgents, } from "../setup/agents.js";
-import { CANONICAL_IMPORT_MAX_BYTES } from "./canonical.js";
-import { uploadAgentSession } from "./client.js";
+import { groupCodexPages } from "./codex-pages.js";
 import { discoverSessions, normalizeRepositoryUrl, repositoryIdentity } from "./discovery.js";
 import { sessionImportFormat } from "./formats.js";
-import { pruneSession } from "./prune.js";
-import { readReceipts, receiptKey, saveReceipt } from "./receipts.js";
+import { readReceipts } from "./receipts.js";
+import { formatBytes, prepareSession, uploadPreparedSession, } from "./sync.js";
 const importAgentIds = SESSION_IMPORT_AGENTS.map((agent) => agent.id);
 const importAgentChoices = importAgentIds.join("|");
 const importAgentDescription = importAgentIds.join(", ");
@@ -137,15 +136,6 @@ export function parseImportArguments(argv) {
         help,
     };
 }
-function formatBytes(bytes) {
-    if (bytes < 1024)
-        return `${bytes} B`;
-    if (bytes < 1024 ** 2)
-        return `${(bytes / 1024).toFixed(1)} KB`;
-    if (bytes < 1024 ** 3)
-        return `${(bytes / 1024 ** 2).toFixed(1)} MB`;
-    return `${(bytes / 1024 ** 3).toFixed(2)} GB`;
-}
 function skippedCounts(skipped) {
     const result = {};
     for (const item of skipped)
@@ -207,7 +197,7 @@ async function runAgentImport(args, agent, context, interactive) {
     const home = resolve(context.env[format.homeEnvironmentVariable] ||
         resolve(context.homeDir, format.defaultHomeDirectory));
     const repository = args.allRepositories ? undefined : repositoryIdentity(context.repoDir);
-    const discovery = discoverSessions({
+    const discovery = await discoverSessions({
         home,
         format,
         repository,
@@ -220,6 +210,20 @@ async function runAgentImport(args, agent, context, interactive) {
         platform: context.platform,
         workingDirectory: context.repoDir,
     });
+    const grouped = format.agent === "codex"
+        ? await groupCodexPages({
+            candidates: discovery.included,
+            metadataByPath: discovery.metadataByPath,
+        })
+        : {
+            candidates: discovery.included.map((candidate) => ({
+                candidate,
+                pages: [candidate.path],
+                merged: false,
+            })),
+            failed: [],
+            cleanup: () => { },
+        };
     try {
         const apiKey = args.apiKey ?? context.env.GROUNDTRACK_IMPORT_KEY ?? context.env.GROUNDTRACK_API_KEY;
         const providedReceiptScope = args.receiptScope ?? context.env.GROUNDTRACK_IMPORT_RECEIPT_SCOPE;
@@ -229,31 +233,21 @@ async function runAgentImport(args, agent, context, interactive) {
         const stableReceiptScope = apiKey?.startsWith("gti_") ? providedReceiptScope : undefined;
         const receipts = apiKey ? readReceipts(context) : { version: 1, receipts: {} };
         const prepared = [];
-        const preparationErrors = [...discovery.failed];
-        for (const candidate of discovery.included) {
+        const preparationErrors = [
+            ...discovery.failed,
+            ...grouped.failed,
+        ];
+        for (const { candidate } of grouped.candidates) {
             try {
-                const pruned = await pruneSession(candidate.path, format, {
-                    emitBytes: false,
-                    metadata: candidate.metadata,
-                });
-                if (pruned.prunedBytes > CANONICAL_IMPORT_MAX_BYTES)
-                    throw new Error(`Pruned session exceeds ${formatBytes(CANONICAL_IMPORT_MAX_BYTES)}`);
-                const key = apiKey
-                    ? receiptKey({
-                        apiBaseUrl: args.apiBaseUrl,
-                        apiKey,
-                        receiptScope: stableReceiptScope,
-                        source: format.source,
-                        sourceContentHash: pruned.sourceContentHash,
-                        visibility: args.visibility,
-                    })
-                    : null;
-                prepared.push({
-                    ...pruned,
+                prepared.push(await prepareSession({
                     candidate,
-                    receiptKey: key,
-                    alreadyImported: Boolean(key && receipts.receipts[key]),
-                });
+                    format,
+                    apiBaseUrl: args.apiBaseUrl,
+                    apiKey: apiKey ?? null,
+                    receiptScope: stableReceiptScope,
+                    visibility: args.visibility,
+                    receipts,
+                }));
             }
             catch (error) {
                 preparationErrors.push({
@@ -324,22 +318,19 @@ async function runAgentImport(args, agent, context, interactive) {
                 continue;
             }
             try {
-                const session = await pruneSession(item.candidate.path, format, {
-                    plan: item,
-                    metadata: item.candidate.metadata,
-                });
-                const result = await uploadAgentSession({
+                const result = await uploadPreparedSession({
+                    prepared: item,
+                    format,
                     apiBaseUrl: args.apiBaseUrl,
                     apiKey,
-                    source: format.source,
-                    session,
                     visibility: args.visibility,
+                    context,
+                    receipts,
                 });
                 if (result.duplicate)
                     duplicate += 1;
                 else
                     imported += 1;
-                saveReceipt(context, receipts, item.receiptKey, result.sessionId);
             }
             catch (error) {
                 failures.push({
@@ -365,6 +356,7 @@ async function runAgentImport(args, agent, context, interactive) {
         return true;
     }
     finally {
+        grouped.cleanup();
         discovery.cleanup?.();
     }
 }

package/dist/session-import/codex-pages.d.ts

@@ -0,0 +1,22 @@
+import type { AgentSessionCandidate, AgentSessionMetadata, FailedAgentSession } from "./types.js";
+export type GroupedCodexCandidate = Readonly<{
+    candidate: AgentSessionCandidate;
+    pages: readonly string[];
+    merged: boolean;
+}>;
+export type GroupedCodexCandidates = Readonly<{
+    candidates: GroupedCodexCandidate[];
+    failed: FailedAgentSession[];
+    cleanup(): void;
+}>;
+/**
+ * Codex can split one logical session across several rollout pages. Each page is
+ * a delta, not a snapshot, so importing pages independently produces several
+ * dashboard rows and a fragment for the newest one. This groups the pages by
+ * `session_id`, walks the `history_base.thread_id` chain, and returns one merged
+ * canonical candidate for the whole session.
+ */
+export declare function groupCodexPages(input: {
+    candidates: readonly AgentSessionCandidate[];
+    metadataByPath: ReadonlyMap<string, AgentSessionMetadata>;
+}): Promise<GroupedCodexCandidates>;

package/dist/session-import/codex-pages.js

@@ -0,0 +1,184 @@
+import { createReadStream, createWriteStream, mkdtempSync, rmSync } from "node:fs";
+import { tmpdir } from "node:os";
+import { basename, join } from "node:path";
+import { createInterface } from "node:readline";
+import { once } from "node:events";
+import { finished } from "node:stream/promises";
+import { isSessionRecord } from "./canonical.js";
+import { createSessionDecompressor, isCompressedSession } from "./compression.js";
+function pageId(path, externalSessionId) {
+    const name = basename(path).replace(/\.jsonl(\.zst)?$/i, "");
+    const suffix = /_([0-9a-f-]{36})$/i.exec(name)?.[1];
+    return suffix ?? externalSessionId;
+}
+function orderedChain(pages) {
+    if (pages.length === 0)
+        return null;
+    const roots = pages.filter((page) => page.baseThreadId === null);
+    if (roots.length !== 1)
+        return null;
+    const children = new Map();
+    for (const page of pages) {
+        if (page.baseThreadId === null)
+            continue;
+        const list = children.get(page.baseThreadId) ?? [];
+        list.push(page);
+        children.set(page.baseThreadId, list);
+    }
+    if ([...children.values()].some((list) => list.length !== 1))
+        return null;
+    const ordered = [];
+    const seen = new Set();
+    let current = roots[0];
+    while (current) {
+        if (seen.has(current.pageId))
+            return null;
+        seen.add(current.pageId);
+        ordered.push(current);
+        current = children.get(current.pageId)?.[0];
+    }
+    return ordered.length === pages.length ? ordered : null;
+}
+async function writeLine(output, line) {
+    if (!output.write(`${line}\n`))
+        await once(output, "drain");
+}
+async function mergePages(chain, outputPath) {
+    const output = createWriteStream(outputPath, { mode: 0o600 });
+    try {
+        for (const [index, page] of chain.entries()) {
+            const input = createReadStream(page.path);
+            const source = isCompressedSession(page.path)
+                ? input.pipe(createSessionDecompressor())
+                : input;
+            input.on("error", (error) => source.destroy(error));
+            const lines = createInterface({ input: source, crlfDelay: Infinity });
+            try {
+                for await (const line of lines) {
+                    if (!line.trim())
+                        continue;
+                    // Only the root page's session_meta is authoritative. Continuation
+                    // pages repeat the header; keeping it would duplicate metadata records.
+                    if (index > 0) {
+                        try {
+                            const record = JSON.parse(line);
+                            if (isSessionRecord(record) && record.type === "session_meta")
+                                continue;
+                        }
+                        catch {
+                            // Keep malformed lines so pruning can classify them.
+                        }
+                    }
+                    await writeLine(output, line);
+                }
+            }
+            finally {
+                lines.close();
+                source.destroy();
+                input.destroy();
+            }
+        }
+    }
+    finally {
+        output.end();
+        await finished(output);
+    }
+}
+/**
+ * Codex can split one logical session across several rollout pages. Each page is
+ * a delta, not a snapshot, so importing pages independently produces several
+ * dashboard rows and a fragment for the newest one. This groups the pages by
+ * `session_id`, walks the `history_base.thread_id` chain, and returns one merged
+ * canonical candidate for the whole session.
+ */
+export async function groupCodexPages(input) {
+    const candidates = [];
+    const includedBySession = new Map();
+    for (const candidate of input.candidates) {
+        if (candidate.metadata.isSubagent) {
+            candidates.push({ candidate, pages: [candidate.path], merged: false });
+            continue;
+        }
+        const list = includedBySession.get(candidate.metadata.externalSessionId) ?? [];
+        list.push(candidate);
+        includedBySession.set(candidate.metadata.externalSessionId, list);
+    }
+    const allPathsBySession = new Map();
+    for (const [path, metadata] of input.metadataByPath) {
+        if (metadata.isSubagent)
+            continue;
+        const list = allPathsBySession.get(metadata.externalSessionId) ?? [];
+        list.push(path);
+        allPathsBySession.set(metadata.externalSessionId, list);
+    }
+    const failed = [];
+    let temporaryDirectory = null;
+    for (const [externalSessionId, included] of includedBySession) {
+        const allPaths = allPathsBySession.get(externalSessionId) ?? [];
+        if (allPaths.length !== included.length) {
+            failed.push({
+                path: included[0].path,
+                error: "Codex session has rollout pages outside this import scope or still live; wait for the session to finish or widen the scope.",
+            });
+            continue;
+        }
+        if (included.length === 1) {
+            if (included[0].metadata.historyBaseThreadId) {
+                failed.push({
+                    path: included[0].path,
+                    error: "Codex session page chain is incomplete; the base page is missing.",
+                });
+                continue;
+            }
+            candidates.push({
+                candidate: included[0],
+                pages: [included[0].path],
+                merged: false,
+            });
+            continue;
+        }
+        const pages = included.map((candidate) => ({
+            path: candidate.path,
+            pageId: pageId(candidate.path, externalSessionId),
+            baseThreadId: candidate.metadata.historyBaseThreadId ?? null,
+        }));
+        const chain = orderedChain(pages);
+        if (!chain) {
+            failed.push({
+                path: included[0].path,
+                error: "Codex session page chain is incomplete or branched.",
+            });
+            continue;
+        }
+        temporaryDirectory ??= mkdtempSync(join(tmpdir(), "groundtrack-codex-"));
+        const mergedPath = join(temporaryDirectory, `${externalSessionId.replaceAll(/[^a-zA-Z0-9-]/g, "_")}.jsonl`);
+        try {
+            await mergePages(chain, mergedPath);
+        }
+        catch (error) {
+            failed.push({
+                path: included[0].path,
+                error: error instanceof Error ? error.message : String(error),
+            });
+            continue;
+        }
+        const root = included.find((candidate) => candidate.path === chain[0].path);
+        candidates.push({
+            candidate: {
+                path: mergedPath,
+                metadata: root.metadata,
+                modifiedAt: new Date(Math.max(...included.map((candidate) => candidate.modifiedAt?.getTime() ?? 0))),
+            },
+            pages: chain.map((page) => page.path),
+            merged: true,
+        });
+    }
+    return {
+        candidates,
+        failed,
+        cleanup: () => {
+            if (temporaryDirectory)
+                rmSync(temporaryDirectory, { recursive: true, force: true });
+        },
+    };
+}

package/dist/session-import/codex.js

@@ -164,6 +164,7 @@ function discoveryMetadata(record) {
         return null;
     const timestampValue = sessionString(payload.timestamp) ?? sessionString(record.timestamp);
     const timestamp = timestampValue ? new Date(timestampValue) : null;
+    const historyBase = isSessionRecord(payload.history_base) ? payload.history_base : null;
     return {
         externalSessionId,
         timestamp: timestamp && !Number.isNaN(timestamp.getTime()) ? timestamp : null,
@@ -173,6 +174,8 @@ function discoveryMetadata(record) {
         repositoryCommitSha: sessionString(git.commit_hash),
         originator: sessionString(payload.originator),
         isSubagent: isSubagentMetadata(payload),
+        historyBaseThreadId: sessionString(historyBase?.thread_id),
+        historyMode: sessionString(payload.history_mode),
     };
 }
 export const codexSessionFormat = {

package/dist/session-import/compression.d.ts

@@ -0,0 +1,8 @@
+import { Transform } from "node:stream";
+/**
+ * Codex compresses sessions older than a few weeks into `.jsonl.zst`. The file is
+ * a standard zstd frame containing the same JSONL records, so decompressing
+ * before hashing keeps the content hash identical to an uncompressed copy.
+ */
+export declare function isCompressedSession(path: string): boolean;
+export declare function createSessionDecompressor(): Transform;

package/dist/session-import/compression.js

@@ -0,0 +1,16 @@
+import { Transform } from "node:stream";
+import * as zlib from "node:zlib";
+/**
+ * Codex compresses sessions older than a few weeks into `.jsonl.zst`. The file is
+ * a standard zstd frame containing the same JSONL records, so decompressing
+ * before hashing keeps the content hash identical to an uncompressed copy.
+ */
+export function isCompressedSession(path) {
+    return path.toLowerCase().endsWith(".jsonl.zst");
+}
+export function createSessionDecompressor() {
+    const create = zlib.createZstdDecompress;
+    if (typeof create !== "function")
+        throw new Error("Reading .jsonl.zst sessions requires Node 22.19 or newer.");
+    return create();
+}

package/dist/session-import/discovery.d.ts

@@ -1,5 +1,14 @@
 import type { SessionImportFormat } from "./format.js";
-import type { AgentSessionCandidate, FailedAgentSession, SkippedAgentSession } from "./types.js";
+import type { AgentSessionCandidate, AgentSessionMetadata, FailedAgentSession, SkippedAgentSession } from "./types.js";
+export type SessionFileStat = Readonly<{
+    size: number;
+    mtimeMs: number;
+}>;
+/**
+ * A metadata-only snapshot of the session files. This is the cheap half of a
+ * watch pass: it walks directories and calls stat, but never opens a session.
+ */
+export declare function statSessionFiles(format: SessionImportFormat, home: string): Map<string, SessionFileStat>;
 export declare function normalizeRepositoryUrl(value: string): string | null;
 export type RepositoryIdentity = Readonly<{
     root: string;
@@ -16,6 +25,11 @@ export type DiscoverSessionsOptions = Readonly<{
     since: Date | null;
     nowMs?: number;
     liveWindowMs?: number;
+    /**
+     * Metadata previously read from a file whose (size, mtime) is unchanged.
+     * The caller is responsible for invalidating entries when either changes.
+     */
+    metadataCache?: ReadonlyMap<string, AgentSessionMetadata>;
     resolveRepositoryRoot?: (path: string) => string;
     resolveRepositoryIdentity?: (path: string) => RepositoryIdentity;
     userHome?: string;
@@ -23,9 +37,12 @@ export type DiscoverSessionsOptions = Readonly<{
     platform?: NodeJS.Platform;
     workingDirectory?: string;
 }>;
-export declare function discoverSessions(options: DiscoverSessionsOptions): {
+export declare function discoverSessions(options: DiscoverSessionsOptions): Promise<{
     included: AgentSessionCandidate[];
     skipped: SkippedAgentSession[];
     failed: FailedAgentSession[];
+    metadataByPath: Map<string, AgentSessionMetadata>;
+    cacheHits: number;
+    cacheMisses: number;
     cleanup?: () => void;
-};
+}>;