@wirecat/cli-core 0.344.0 → 0.346.0

package/dist/services/project-gathering.js

@@ -0,0 +1,202 @@
+import { CliError } from "@wirecat/cli-base";
+import { createContextEvidenceReader } from "./context-evidence.js";
+import { excerpt } from "./excerpt.js";
+import { decodeCheckpoint, encodeCheckpoint, projectGatheringRequest, projectUnits, seenKey, } from "./project-gathering-request.js";
+const terminal = new Set(["done", "unavailable", "failed"]);
+export const gatherProjectEvidence = async (ports, request) => {
+    if (!ports?.evidence || typeof ports.evidence.read !== "function")
+        throw new CliError("configuration_error", "stored context reads are unavailable in this store");
+    const now = ports.now ?? Date.now;
+    const started = now();
+    const plan = projectGatheringRequest(request, started);
+    const previous = plan.checkpoint === undefined ? null : decodeCheckpoint(plan.checkpoint, plan);
+    const window = previous ?? plan.window;
+    const units = projectUnits(plan, window);
+    if (previous && (previous.units.length !== units.length || previous.fetched.length !== plan.sources.length))
+        throw new CliError("validation_error", "invalid project gathering checkpoint", { param: "checkpoint" });
+    const progress = previous?.units.map((u) => ({ ...u })) ??
+        units.map(() => ({ state: "pending", pages: 0, cursor: null, code: null }));
+    const fetched = previous?.fetched.map((f) => ({ ...f })) ??
+        plan.sources.map(() => ({ state: "not-requested", history: null, code: null }));
+    const seen = new Set(previous?.seen ?? []);
+    const signal = plan.signal;
+    if (plan.fetch === "before-read" && !ports.fetcher)
+        throw new CliError("configuration_error", "this host cannot fetch source history; use fetch none", {
+            param: "fetch",
+        });
+    if (plan.fetch === "before-read" && ports.fetcher)
+        for (const [index, source] of plan.sources.entries()) {
+            if (fetched[index]?.state !== "not-requested")
+                continue;
+            signal?.throwIfAborted();
+            try {
+                const answer = await ports.fetcher.fetch({ source, after: window.after, before: window.before, signal });
+                if (!["fetched", "unsupported"].includes(answer?.status) ||
+                    !["complete", "partial", "unknown"].includes(answer?.history))
+                    throw new CliError("invalid_response", "source fetch returned an unknown status");
+                fetched[index] = { state: answer.status, history: answer.history, code: null };
+            }
+            catch (error) {
+                signal?.throwIfAborted();
+                if (!(error instanceof CliError))
+                    throw error;
+                fetched[index] = { state: "failed", history: null, code: error.code };
+            }
+        }
+    const usage = { reads: 0, readBytes: 0, items: 0 };
+    const items = [];
+    const byRef = new Map();
+    const matchedAgain = new Map();
+    let stopped = null;
+    const open = (u) => !terminal.has(u.state) && u.state !== "read-limited";
+    const take = (item, lane, kind, source) => {
+        const known = byRef.get(item.ref);
+        if (known) {
+            if (!known.lanes.includes(lane.id))
+                known.lanes.push(lane.id);
+            if (!lane.kinds.every((k) => k === "person"))
+                known.context = "project";
+            return;
+        }
+        const key = seenKey(item.ref);
+        if (seen.has(key)) {
+            matchedAgain.set(item.ref, (matchedAgain.get(item.ref) ?? new Set()).add(lane.id));
+            return;
+        }
+        seen.add(key);
+        const short = excerpt(item.text, lane.query, 300, { literal: true });
+        const entry = {
+            ...item,
+            kind,
+            source,
+            text: short,
+            textTruncated: short !== item.text.replace(/\s+/g, " ").trim(),
+            lanes: [lane.id],
+            context: lane.kinds.every((k) => k === "person") ? "person-only" : "project",
+        };
+        byRef.set(item.ref, entry);
+        items.push(entry);
+        usage.items++;
+    };
+    for (let round = 0; !stopped; round++) {
+        const due = progress.flatMap((u, index) => (open(u) && u.pages <= round ? [index] : []));
+        if (!due.length)
+            break;
+        for (const index of due) {
+            const unit = units[index], state = progress[index];
+            if (!unit || !state)
+                continue;
+            signal?.throwIfAborted();
+            stopped =
+                usage.reads >= plan.budgets.maxReads
+                    ? "reads"
+                    : usage.readBytes >= plan.budgets.maxReadBytes
+                        ? "read-bytes"
+                        : now() - started >= plan.budgets.maxElapsedMs
+                            ? "elapsed"
+                            : usage.items >= plan.budgets.maxItems
+                                ? "items"
+                                : null;
+            if (stopped)
+                break;
+            const allowance = Math.min(plan.budgets.maxPageBytes, plan.budgets.maxReadBytes - usage.readBytes);
+            const reader = createContextEvidenceReader(ports.evidence, [unit.source], {
+                maxReads: 1,
+                maxTotalReadBytes: allowance,
+            });
+            usage.reads++;
+            try {
+                const page = await reader.read({
+                    source: unit.source,
+                    query: unit.lane.query,
+                    after: unit.chunk.after,
+                    before: unit.chunk.before,
+                    limit: 20,
+                    maxReadBytes: allowance,
+                    ...(state.cursor ? { cursor: state.cursor } : {}),
+                    signal,
+                });
+                usage.readBytes += page.readBytes;
+                state.pages++;
+                state.code = page.coverage.index === "ready" ? null : `index-${page.coverage.index}`;
+                for (const item of page.items)
+                    take(item, unit.lane, unit.source.kind, unit.sourceIndex);
+                if (!page.coverage.available)
+                    state.state = "unavailable";
+                else if (page.next) {
+                    state.state = "partial";
+                    state.cursor = page.next;
+                }
+                else {
+                    state.state = "done";
+                    state.cursor = null;
+                }
+            }
+            catch (error) {
+                signal?.throwIfAborted();
+                usage.readBytes += reader.usage().chargedBytes;
+                if (!(error instanceof CliError))
+                    throw error;
+                if (error.code === "validation_error" && error.details?.reason === "context.read-budget")
+                    state.state = "read-limited";
+                else
+                    state.state = error.code === "not_found" ? "unavailable" : "failed";
+                state.code = error.code;
+            }
+        }
+    }
+    const totals = {
+        reads: (previous?.totals.reads ?? 0) + usage.reads,
+        readBytes: (previous?.totals.readBytes ?? 0) + usage.readBytes,
+        items: (previous?.totals.items ?? 0) + usage.items,
+        invocations: (previous?.totals.invocations ?? 0) + 1,
+    };
+    const resumable = progress.some((u) => !terminal.has(u.state));
+    const checkpoint = {
+        version: 1,
+        runKey: plan.runKey,
+        fingerprint: plan.fingerprint,
+        after: window.after,
+        before: window.before,
+        asOf: window.asOf,
+        fetched,
+        units: progress.map((u) => (u.state === "read-limited" ? { ...u, state: "partial" } : u)),
+        seen: [...seen],
+        totals,
+    };
+    return {
+        schemaVersion: 1,
+        runKey: plan.runKey,
+        fingerprint: plan.fingerprint,
+        resumed: previous !== null,
+        project: plan.project,
+        window: {
+            after: window.after,
+            before: window.before,
+            asOf: window.asOf,
+            lookbackDays: plan.lookbackDays,
+            basis: plan.lookbackDays === null ? "explicit" : "lookback",
+            timeBasis: "per-item-time-kind",
+        },
+        scope: { origin: plan.scope, sources: plan.sources },
+        lanes: plan.lanes,
+        fetch: fetched.map((f, index) => ({ source: index, ...f })),
+        units: units.map((unit, index) => ({
+            lane: unit.lane.id,
+            source: unit.sourceIndex,
+            after: unit.chunk.after,
+            before: unit.chunk.before,
+            state: progress[index]?.state ?? "pending",
+            pages: progress[index]?.pages ?? 0,
+            issue: progress[index]?.code ?? null,
+        })),
+        items,
+        matchedAgain: [...matchedAgain].map(([ref, lanes]) => ({ ref, lanes: [...lanes] })),
+        usage: { ...usage, elapsedMs: now() - started },
+        totals,
+        stopped,
+        complete: !resumable,
+        completeness: "unknown",
+        checkpoint: resumable ? encodeCheckpoint(checkpoint) : null,
+    };
+};

package/dist/store/sqlite/chats.js

@@ -4,7 +4,7 @@ import { identityPk } from "./identities.js";
 import { bound } from "./query-filter.js";
 import { chatEventPeople, chatEvents, chatMembers, chats, chunks, fetchLeases, identities, involvements, memberCounts, memberObservationMembers, memberObservations, memberStays, messageRevisions, messages, messageTranscripts, pollVotes, syncRanges, } from "./schema.js";
 import { ensureTag } from "./tags.js";
-import { json, parsed, present, toIso, toMs } from "./values.js";
+import { parsed, present, toIso, toMs } from "./values.js";
 /** The chat's own id, when the store key is a chat of this account. */
 export const chatOf = ({ orm }, accountKey, chatKey) => orm
     .select({ id: chats.externalId })
@@ -31,7 +31,12 @@ export const chatPkFor = (context, accountKey, chatId) => findChatPk(context, ac
         .get()?.pk);
 export const upsertChat = (context, accountKey, chat) => {
     const { orm, now } = context;
-    const parent = chat.parentChatId === undefined ? null : chatPkFor(context, accountKey, chat.parentChatId);
+    const parent = chat.parentChatId == null ? null : chatPkFor(context, accountKey, chat.parentChatId);
+    const stored = orm
+        .select({ metadata: chats.metadata })
+        .from(chats)
+        .where(and(eq(chats.accountId, accountKey), eq(chats.externalId, chat.id)))
+        .get();
     orm
         .insert(chats)
         .values({
@@ -44,7 +49,7 @@ export const upsertChat = (context, accountKey, chat) => {
         unreadCount: chat.unreadCount,
         lastMessageAt: toMs(chat.lastMessageAt),
         participantsCount: chat.participantsCount,
-        metadata: json(chat.providerMetadata),
+        metadata: mergedMetadata(stored?.metadata ?? null, chat.providerMetadata),
         membershipState: chat.membershipState ?? null,
         createdAt: now(),
         updatedAt: now(),
@@ -53,7 +58,7 @@ export const upsertChat = (context, accountKey, chat) => {
         target: [chats.accountId, chats.externalId],
         set: {
             kind: sql `excluded.kind`,
-            parentChatId: sql `coalesce(excluded.parent_chat_id, ${chats.parentChatId})`,
+            ...(chat.parentChatId === undefined ? {} : { parentChatId: sql `excluded.parent_chat_id` }),
             scope: sql `coalesce(excluded.scope, ${chats.scope})`,
             title: sql `excluded.title`,
             unreadCount: sql `excluded.unread_count`,
@@ -68,6 +73,22 @@ export const upsertChat = (context, accountKey, chat) => {
     })
         .run();
 };
+const mergedMetadata = (stored, incoming) => {
+    if (incoming === undefined)
+        return stored;
+    const merged = Object.entries({ ...metadataObject(stored), ...incoming }).filter(([, value]) => value !== null);
+    return merged.length === 0 ? null : JSON.stringify(Object.fromEntries(merged));
+};
+// Stored metadata that is not an object has no keys to keep.
+const metadataObject = (stored) => {
+    try {
+        const value = stored === null ? null : JSON.parse(stored);
+        return value !== null && typeof value === "object" && !Array.isArray(value) ? value : {};
+    }
+    catch {
+        return {};
+    }
+};
 /** Replaces who is in the chat, whole. */
 export const writeMembers = (context, key, accountKey, chatId, memberIds) => {
     const chatKey = chatPkFor(context, accountKey, chatId);

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@wirecat/cli-core",
-  "version": "0.344.0",
+  "version": "0.346.0",
   "description": "The messenger-neutral half of a messaging CLI: domain model, message locators, rendering, name resolution, the SQLite seam, the command skeleton and shared read commands, the send guard, run records and the message store",
   "license": "Apache-2.0",
   "author": "Viacheslav Ptsarev",
@@ -158,7 +158,8 @@
     "standards:check": "node config/wirecat/check.mjs",
     "docs:lint": "markdownlint-cli2",
     "docs:spell": "cspell lint --config cspell.json --no-progress README.md",
-    "ci:quick": "pnpm typecheck && vitest run src/cli/command-lint.test.ts"
+    "ci:quick": "pnpm typecheck && vitest run src/cli/command-lint.test.ts",
+    "help:examples": "HELP_EXAMPLES_REPORT=1 vitest run scripts/help-examples.test.ts --silent=false"
   },
   "dependencies": {
     "@anthropic-ai/sdk": "0.131.0",
@@ -201,7 +202,7 @@
     "@types/node": "^22.20.4",
     "@types/pngjs": "6.0.5",
     "@vitest/coverage-v8": "^5.0.2",
-    "@wirecat/cli-base": "0.45.0",
+    "@wirecat/cli-base": "0.46.0",
     "commander": "15.0.0",
     "cspell": "10.3.6",
     "drizzle-kit": "1.0.0-rc.4",
@@ -221,7 +222,7 @@
   },
   "peerDependencies": {
     "@napi-rs/canvas": ">=0.1.69 <2",
-    "@wirecat/cli-base": ">=0.45.0 <0.46.0",
+    "@wirecat/cli-base": ">=0.45.0 <0.47.0",
     "commander": ">=15"
   },
   "bin": {

package/parity.json

@@ -133,7 +133,7 @@
       "meaning": "also read these bots' copies, comma separated — each allowed by read_other_bots"
     },
     "--brief": {
-      "meaning": "found messages and mail as reference, title, time, sender and excerpt only"
+      "meaning": "found messages as reference, title, time, sender and excerpt only"
     },
     "--by": {
       "value": "<grouping>",
@@ -394,7 +394,7 @@
       "meaning": "read nonrecursive files from this directory for one explicit chat"
     },
     "--full": {
-      "meaning": "found messages and mail with their whole text and every provider field"
+      "meaning": "found messages with their whole text and every provider field"
     },
     "--hidden": {
       "value": "<on|off>",