@herbertgao/pi-extensions 2026.8.11 → 2026.8.13

package/node_modules/@luxusai/pi-hindsight/extensions/queue/queue-operations.ts

@@ -0,0 +1,70 @@
+import {
+  resolveQueuePath,
+  summarizeRetainQueue,
+  readRetainQueueTolerant,
+  readDeadLetterQueueTolerant,
+} from "./queue.js";
+import type { MemoryOperationsDeps } from "../operations/memory-operation-types.js";
+import { redactError } from "../utils/sanitize.js";
+import { listRetainReceipts } from "../lifecycle/retain-receipts.js";
+import type { RetainJob } from "../types.js";
+
+const RECENT_OUTCOME_LIMIT = 5;
+
+function redactJob(job: RetainJob) {
+  return {
+    id: job.id,
+    bankId: job.bankId,
+    documentId: job.documentId,
+    updateMode: job.updateMode,
+    retries: job.retries,
+    lastError: job.lastError ? redactError(job.lastError) : undefined,
+    deadLetteredAt: job.deadLetteredAt,
+    tags: job.item.tags,
+    metadataKeys: job.item.metadata ? Object.keys(job.item.metadata).sort() : undefined,
+    contentBytes: Buffer.byteLength(job.item.content, "utf8"),
+    contextBytes: Buffer.byteLength(job.item.context, "utf8"),
+  };
+}
+
+export function createQueueOperations(deps: MemoryOperationsDeps) {
+  return {
+    async inspectRetainQueue(args: { cwd: string; includeJobs?: boolean }) {
+      const config = deps.getConfig();
+      const queuePath = resolveQueuePath(args.cwd, config.retain.queuePath);
+      const summary = await summarizeRetainQueue(queuePath);
+      const activeJobs = args.includeJobs
+        ? await readRetainQueueTolerant(queuePath)
+            .then((parsed) => parsed.jobs)
+            .catch(() => [])
+        : [];
+      const deadLetterJobs = args.includeJobs
+        ? await readDeadLetterQueueTolerant(queuePath)
+            .then((parsed) => parsed.jobs)
+            .catch(() => [])
+        : [];
+      const recentOutcomes = (await listRetainReceipts(args.cwd, RECENT_OUTCOME_LIMIT)).map(
+        (receipt) => ({
+          documentId: receipt.documentId,
+          bankId: receipt.bankId,
+          source: receipt.source,
+          createdAt: receipt.createdAt,
+          ...(receipt.outcome ? { outcome: receipt.outcome } : {}),
+        }),
+      );
+      return {
+        queuePath,
+        deadLetterPath: `${queuePath}.dead.jsonl`,
+        active: summary.active,
+        deadLetter: summary.deadLetter,
+        recentOutcomes,
+        jobs: args.includeJobs
+          ? {
+              active: activeJobs.map(redactJob),
+              deadLetter: deadLetterJobs.map(redactJob),
+            }
+          : undefined,
+      };
+    },
+  };
+}

package/node_modules/@luxusai/pi-hindsight/extensions/queue/queue.ts

@@ -0,0 +1,392 @@
+import { appendFile, mkdir } from "node:fs/promises";
+import { dirname, isAbsolute, join } from "node:path";
+import type {
+  HindsightLikeClient,
+  ResolvedConfig,
+  RetainJob,
+  RetainOutcome,
+  UpdateMode,
+} from "../types.js";
+import { deliverRetainJob, parseRetainOutcome, redactQueueError } from "./queue-delivery.js";
+import { JsonlQueueStore, type JsonlQueueFileSummary } from "./jsonl-queue-store.js";
+import { withQueueLock } from "./queue-lock.js";
+
+export { RETAIN_QUEUE_LOCK, isQueueLockOwnerStale, type QueueLockOwner } from "./queue-lock.js";
+
+export function resolveQueuePath(cwd: string, queuePath: string): string {
+  return isAbsolute(queuePath) ? queuePath : join(cwd, queuePath);
+}
+
+export function retainQueuePath(cwd: string, config: ResolvedConfig): string {
+  return resolveQueuePath(cwd, config.retain.queuePath);
+}
+
+export function resolveDeadLetterQueuePath(path: string): string {
+  return `${path}.dead.jsonl`;
+}
+
+export function resolveMalformedQueuePath(path: string): string {
+  return `${path}.malformed.jsonl`;
+}
+
+export interface EnqueueRetainJobResult {
+  previousLength: number;
+  currentLength: number;
+}
+
+export async function enqueueRetainJobWithStats(
+  path: string,
+  job: RetainJob,
+): Promise<EnqueueRetainJobResult> {
+  return withQueueLock(path, async () => {
+    const store = retainQueueStore(path);
+    const previousLength = await store.count();
+    await store.append([job]);
+    return { previousLength, currentLength: previousLength + 1 };
+  });
+}
+
+export async function enqueueRetainJob(path: string, job: RetainJob): Promise<void> {
+  await enqueueRetainJobWithStats(path, job);
+}
+
+// Two automatic-retain jobs can be merged into one remote operation when they target the
+// same bank + document with append semantics and carry no delivery failures yet. Merging
+// concatenates their (cursor-filtered) deltas so a single delivery covers both, which is
+// what cuts server extraction/consolidation and Postgres write amplification.
+export function canCoalesceRetainJobs(existing: RetainJob, incoming: RetainJob): boolean {
+  return (
+    existing.bankId === incoming.bankId &&
+    existing.documentId === incoming.documentId &&
+    existing.updateMode === "append" &&
+    incoming.updateMode === "append" &&
+    existing.retries === 0 &&
+    existing.deadLetteredAt === undefined
+  );
+}
+
+function mergeRetainContent(existing: string, incoming: string): string {
+  // Automatic-retain content is a JSON array of projected messages. When both sides parse
+  // as arrays, concatenate the elements so the merged job is indistinguishable from having
+  // enqueued the deltas sequentially. Fall back to newline concatenation otherwise.
+  try {
+    const a = JSON.parse(existing);
+    const b = JSON.parse(incoming);
+    if (Array.isArray(a) && Array.isArray(b)) return JSON.stringify([...a, ...b], null, 2);
+  } catch {
+    // fall through to string concatenation
+  }
+  return `${existing}\n${incoming}`;
+}
+
+export function coalesceRetainJob(existing: RetainJob, incoming: RetainJob): RetainJob {
+  const tags = [...new Set([...(existing.item.tags ?? []), ...(incoming.item.tags ?? [])])];
+  return {
+    ...existing,
+    item: {
+      ...existing.item,
+      ...incoming.item,
+      content: mergeRetainContent(existing.item.content, incoming.item.content),
+      ...(tags.length ? { tags } : {}),
+    },
+  };
+}
+
+// Append the job, or merge into the last queue entry only when compatible.
+// Only the tail is considered so append order never jumps past a failed/non-mergeable job.
+export async function enqueueRetainJobCoalesced(
+  path: string,
+  job: RetainJob,
+): Promise<{ coalesced: boolean; currentLength: number }> {
+  return withQueueLock(path, async () => {
+    const store = retainQueueStore(path);
+    const parsed = await store.readTolerant();
+    const jobs = parsed.jobs;
+    const last = jobs[jobs.length - 1];
+    if (last && canCoalesceRetainJobs(last, job)) {
+      jobs[jobs.length - 1] = coalesceRetainJob(last, job);
+      await appendMalformedQueueLines(path, parsed.malformedLines);
+      await store.replace(jobs);
+      return { coalesced: true, currentLength: jobs.length };
+    }
+    await store.append([job]);
+    return { coalesced: false, currentLength: jobs.length + 1 };
+  });
+}
+
+export async function readRetainQueue(path: string): Promise<RetainJob[]> {
+  return retainQueueStore(path).readStrict();
+}
+
+export async function readDeadLetterQueue(path: string): Promise<RetainJob[]> {
+  return readRetainQueue(resolveDeadLetterQueuePath(path));
+}
+
+export async function readRetainQueueTolerant(path: string) {
+  return retainQueueStore(path).readTolerant();
+}
+
+export async function readDeadLetterQueueTolerant(path: string) {
+  return readRetainQueueTolerant(resolveDeadLetterQueuePath(path));
+}
+
+export type RetainQueueFileSummary = JsonlQueueFileSummary;
+
+export interface RetainQueueSummary {
+  active: RetainQueueFileSummary;
+  deadLetter: RetainQueueFileSummary;
+}
+
+export async function summarizeRetainQueue(path: string): Promise<RetainQueueSummary> {
+  const [active, deadLetter] = await Promise.all([
+    retainQueueStore(path).summarize(),
+    retainQueueStore(resolveDeadLetterQueuePath(path)).summarize(),
+  ]);
+  return { active, deadLetter };
+}
+
+export async function writeRetainQueue(path: string, jobs: RetainJob[]): Promise<void> {
+  await retainQueueStore(path).replace(jobs);
+}
+
+export async function removeRetainQueueJobs(
+  path: string,
+  predicate: (job: RetainJob) => boolean,
+): Promise<number> {
+  return withQueueLock(path, async () => {
+    const parsed = await readRetainQueueTolerant(path);
+    const remaining = parsed.jobs.filter((job) => !predicate(job));
+    const removed = parsed.jobs.length - remaining.length;
+    if (removed > 0) {
+      await appendMalformedQueueLines(path, parsed.malformedLines);
+      await writeRetainQueue(path, remaining);
+    }
+    return removed;
+  });
+}
+
+async function appendDeadLetterJobs(path: string, jobs: RetainJob[]): Promise<number> {
+  if (jobs.length === 0) return 0;
+  const deadLetterPath = resolveDeadLetterQueuePath(path);
+  const existing = await readRetainQueueTolerant(deadLetterPath);
+  const existingIds = new Set(existing.jobs.map((job) => job.id));
+  const seen = new Set<string>();
+  const newJobs = jobs.filter((job) => {
+    if (existingIds.has(job.id) || seen.has(job.id)) return false;
+    seen.add(job.id);
+    return true;
+  });
+  if (newJobs.length === 0) return 0;
+  await retainQueueStore(deadLetterPath).append(newJobs);
+  return newJobs.length;
+}
+
+export type FlushRetainQueueOptions = {
+  maxRetries?: number;
+  maxJobs?: number;
+  stopOnFirstFailure?: boolean;
+  maxElapsedMs?: number;
+};
+
+export interface RetainDeliverySummary {
+  queueJobId: string;
+  bankId: string;
+  documentId: string;
+  updateMode: UpdateMode;
+  context: string;
+  tags: string[];
+  outcome: RetainOutcome;
+}
+
+export interface FlushRetainQueueResult {
+  sent: number;
+  remaining: number;
+  deadLettered: number;
+  malformed: number;
+  operationIds?: string[];
+  outcome?: RetainOutcome;
+  delivered?: RetainDeliverySummary[];
+}
+
+function isRetainJob(value: unknown): value is RetainJob {
+  if (!value || typeof value !== "object") return false;
+  const record = value as Record<string, unknown>;
+  const item = record.item as Record<string, unknown> | undefined;
+  return (
+    typeof record.id === "string" &&
+    typeof record.bankId === "string" &&
+    typeof record.documentId === "string" &&
+    (record.updateMode === "append" || record.updateMode === "replace") &&
+    typeof record.retries === "number" &&
+    !!item &&
+    typeof item === "object" &&
+    typeof item.content === "string" &&
+    typeof item.context === "string"
+  );
+}
+
+function retainQueueStore(path: string): JsonlQueueStore<RetainJob> {
+  return new JsonlQueueStore(path, isRetainJob);
+}
+
+async function appendMalformedQueueLines(path: string, lines: string[]): Promise<void> {
+  if (lines.length === 0) return;
+  const malformedPath = resolveMalformedQueuePath(path);
+  await mkdir(dirname(malformedPath), { recursive: true });
+  const quarantinedAt = new Date().toISOString();
+  await appendFile(
+    malformedPath,
+    lines
+      .map((line) =>
+        JSON.stringify({
+          quarantinedAt,
+          sourceQueue: path,
+          line,
+        }),
+      )
+      .join("\n") + "\n",
+    "utf8",
+  );
+}
+
+export async function flushRetainQueue(
+  path: string,
+  client: HindsightLikeClient,
+  options: FlushRetainQueueOptions = {},
+): Promise<FlushRetainQueueResult> {
+  return withQueueLock(path, async () => {
+    const resolvedOptions = options;
+    const maxRetries = resolvedOptions.maxRetries ?? 5;
+    const maxJobs = resolvedOptions.maxJobs ?? Number.POSITIVE_INFINITY;
+    const maxElapsedMs = resolvedOptions.maxElapsedMs ?? Number.POSITIVE_INFINITY;
+    const started = Date.now();
+    const parsed = await readRetainQueueTolerant(path);
+    const jobs = parsed.jobs;
+    const remaining: RetainJob[] = [];
+    const deadLetteredJobs: RetainJob[] = [];
+    const operationIds: string[] = [];
+    const delivered: RetainDeliverySummary[] = [];
+    let itemsCount = 0;
+    let hasItemsCount = false;
+    let tokens = 0;
+    let hasTokens = false;
+    let sent = 0;
+    for (const [index, job] of jobs.entries()) {
+      if (index >= maxJobs || Date.now() - started >= maxElapsedMs) {
+        remaining.push(...jobs.slice(index));
+        break;
+      }
+      try {
+        const response = await deliverRetainJob(client, job);
+        const outcome = parseRetainOutcome(response);
+        operationIds.push(...outcome.operationIds);
+        if (outcome.itemsCount !== undefined) {
+          itemsCount += outcome.itemsCount;
+          hasItemsCount = true;
+        }
+        if (outcome.tokens !== undefined) {
+          tokens += outcome.tokens;
+          hasTokens = true;
+        }
+        const jobOutcome: RetainOutcome = {
+          ...(outcome.itemsCount !== undefined ? { itemsCount: outcome.itemsCount } : {}),
+          ...(outcome.operationIds.length ? { operations: outcome.operationIds.length } : {}),
+          ...(outcome.tokens !== undefined ? { tokens: outcome.tokens } : {}),
+        };
+        delivered.push({
+          queueJobId: job.id,
+          bankId: job.bankId,
+          documentId: job.documentId,
+          updateMode: job.updateMode,
+          context: job.item.context,
+          tags: job.item.tags ?? [],
+          outcome: jobOutcome,
+        });
+        sent += 1;
+      } catch (error) {
+        const errorMessage = redactQueueError(error);
+        const retries = job.retries + 1;
+        const deadLetter = retries >= maxRetries;
+        const failedJob = {
+          ...job,
+          retries,
+          lastError: errorMessage,
+          ...(deadLetter
+            ? {
+                deadLetteredAt: new Date().toISOString(),
+                lastError: `${errorMessage}; retry limit reached, moved to dead-letter queue`,
+              }
+            : {}),
+        };
+        if (deadLetter) deadLetteredJobs.push(failedJob);
+        else remaining.push(failedJob);
+        if (resolvedOptions.stopOnFirstFailure) {
+          remaining.push(...jobs.slice(index + 1));
+          break;
+        }
+      }
+    }
+    await appendMalformedQueueLines(path, parsed.malformedLines);
+    const appendedDeadLetterJobs = await appendDeadLetterJobs(path, deadLetteredJobs);
+    await writeRetainQueue(path, remaining);
+    const uniqueOperationIds = [...new Set(operationIds)];
+    const aggregate: RetainOutcome = {
+      ...(hasItemsCount ? { itemsCount } : {}),
+      ...(uniqueOperationIds.length ? { operations: uniqueOperationIds.length } : {}),
+      ...(hasTokens ? { tokens } : {}),
+    };
+    return {
+      sent,
+      remaining: remaining.length,
+      deadLettered: appendedDeadLetterJobs,
+      malformed: parsed.malformedLines.length,
+      ...(uniqueOperationIds.length ? { operationIds: uniqueOperationIds } : {}),
+      ...(hasItemsCount || uniqueOperationIds.length || hasTokens ? { outcome: aggregate } : {}),
+      ...(delivered.length ? { delivered } : {}),
+    };
+  });
+}
+
+export async function enqueueRetain(
+  cwd: string,
+  config: ResolvedConfig,
+  job: RetainJob,
+): Promise<EnqueueRetainJobResult> {
+  return enqueueRetainJobWithStats(retainQueuePath(cwd, config), job);
+}
+
+export async function enqueueRetainCoalesced(
+  cwd: string,
+  config: ResolvedConfig,
+  job: RetainJob,
+): Promise<{ coalesced: boolean; currentLength: number }> {
+  return enqueueRetainJobCoalesced(retainQueuePath(cwd, config), job);
+}
+
+export async function flushRetain(
+  cwd: string,
+  config: ResolvedConfig,
+  client: HindsightLikeClient,
+  options?: FlushRetainQueueOptions,
+): Promise<FlushRetainQueueResult> {
+  return flushRetainQueue(retainQueuePath(cwd, config), client, options);
+}
+
+export async function readQueuedRetains(cwd: string, config: ResolvedConfig): Promise<RetainJob[]> {
+  return readRetainQueue(retainQueuePath(cwd, config));
+}
+
+export async function removeQueuedRetains(
+  cwd: string,
+  config: ResolvedConfig,
+  predicate: (job: RetainJob) => boolean,
+): Promise<number> {
+  return removeRetainQueueJobs(retainQueuePath(cwd, config), predicate);
+}
+
+export async function summarizeRetain(
+  cwd: string,
+  config: ResolvedConfig,
+): Promise<RetainQueueSummary> {
+  return summarizeRetainQueue(retainQueuePath(cwd, config));
+}

package/node_modules/@luxusai/pi-hindsight/extensions/tui/bank-template-presentation.ts

@@ -0,0 +1,45 @@
+import type { BankTemplateImportResponse } from "@vectorize-io/hindsight-client";
+import type { BuiltInBankTemplate } from "../banks/bank-templates.js";
+
+export function renderBankTemplateList(templates: readonly BuiltInBankTemplate[]): string {
+  const lines = templates.map((template) => {
+    const mentalModels = template.manifest.mental_models?.length ?? 0;
+    const directives = template.manifest.directives?.length ?? 0;
+    return `- ${template.id} (${template.target}): ${template.label} — ${template.description} mentalModels=${mentalModels} directives=${directives}`;
+  });
+  return ["Hindsight bank templates:", ...lines].join("\n");
+}
+
+function summarizeImportResponse(result: unknown): string {
+  if (typeof result !== "object" || !result) return JSON.stringify(result);
+  const response = result as BankTemplateImportResponse;
+  const created = response.mental_models_created?.length ?? 0;
+  const updated = response.mental_models_updated?.length ?? 0;
+  const directivesCreated = response.directives_created?.length ?? 0;
+  const directivesUpdated = response.directives_updated?.length ?? 0;
+  return `configApplied=${response.config_applied}; mentalModels created=${created}/updated=${updated}; directives created=${directivesCreated}/updated=${directivesUpdated}`;
+}
+
+export function renderBankTemplateApplyResult(args: {
+  bankId: string;
+  template: BuiltInBankTemplate;
+  dryRun: boolean;
+  result: unknown;
+}): string {
+  const prefix = args.dryRun
+    ? `Bank template preview: ${args.template.id} -> ${args.bankId}`
+    : `Applied bank template: ${args.template.id} -> ${args.bankId}`;
+  return `${prefix}; ${summarizeImportResponse(args.result)}; write=${args.dryRun ? "no" : "yes"}`;
+}
+
+// Shows each mental model's name, source query, and tags before submission, per
+// docs/starter-mental-model-suggestions.md's product rules.
+export function renderBankTemplateMentalModelDetails(template: BuiltInBankTemplate): string {
+  const models = template.manifest.mental_models ?? [];
+  if (!models.length) return "";
+  const lines = models.map(
+    (model) =>
+      `- ${model.name} (${model.id}); tags=${model.tags?.join(",") || "none"}; query=${model.source_query}`,
+  );
+  return [`Mental models for ${template.id} (${template.target} bank):`, ...lines].join("\n");
+}

package/node_modules/@luxusai/pi-hindsight/extensions/tui/commands.ts

@@ -0,0 +1,9 @@
+import type { ExtensionAPI } from "@earendil-works/pi-coding-agent";
+import type { MemoryOperationsDeps } from "../operations/memory-operation-service.js";
+import { createOperationCatalog } from "../operations/operation-catalog.js";
+
+export function registerCommands(pi: ExtensionAPI, deps: MemoryOperationsDeps) {
+  for (const command of createOperationCatalog(deps).commands) {
+    pi.registerCommand(command.name, command.spec);
+  }
+}