@herbertgao/pi-extensions 2026.8.11 → 2026.8.13

package/node_modules/@luxusai/pi-hindsight/extensions/lifecycle/retain-job-builder.ts

@@ -0,0 +1,66 @@
+import { randomUUID } from "node:crypto";
+import type {
+  HindsightObservationScopes,
+  ResolvedConfig,
+  RetainJob,
+  UpdateMode,
+} from "../types.js";
+import { redactSecrets } from "../utils/sanitize.js";
+
+export interface RetainJobBuildArgs {
+  config: ResolvedConfig;
+  bankId: string;
+  content: string;
+  context: string;
+  tags: string[];
+  documentId: string;
+  updateMode: UpdateMode;
+  metadata?: Record<string, string>;
+  timestamp?: string;
+  observationScopes?: HindsightObservationScopes;
+  documentTags?: string[];
+  entities?: RetainJob["item"]["entities"];
+  async?: boolean;
+  /** Named bank retain strategy (e.g. conversation, gitlog). Overrides bank default for this item. */
+  strategy?: string;
+}
+
+function sanitizedMetadata(
+  metadata: Record<string, string> | undefined,
+  redact: boolean,
+): Record<string, string> | undefined {
+  if (!metadata) return undefined;
+  if (!redact) return metadata;
+  return Object.fromEntries(
+    Object.entries(metadata).map(([key, value]) => [key, redactSecrets(value)]),
+  );
+}
+
+export function buildRetainJob(args: RetainJobBuildArgs): RetainJob {
+  const redact = args.config.retain.redactSecrets;
+  const content = redact ? redactSecrets(args.content) : args.content;
+  const context = redact ? redactSecrets(args.context) : args.context;
+  const metadata = sanitizedMetadata(args.metadata, redact);
+  return {
+    id: randomUUID(),
+    bankId: args.bankId,
+    createdAt: new Date().toISOString(),
+    documentId: args.documentId,
+    updateMode: args.updateMode,
+    item: {
+      content,
+      context,
+      timestamp: args.timestamp ?? new Date().toISOString(),
+      async: args.async ?? args.config.retain.async,
+      ...((args.entities?.length ?? args.config.retain.entities.length)
+        ? { entities: [...args.config.retain.entities, ...(args.entities ?? [])] }
+        : {}),
+      tags: args.tags,
+      ...(metadata ? { metadata } : {}),
+      ...(args.observationScopes?.length ? { observationScopes: args.observationScopes } : {}),
+      ...(args.documentTags?.length ? { documentTags: args.documentTags } : {}),
+      ...(args.strategy ? { strategy: args.strategy } : {}),
+    },
+    retries: 0,
+  };
+}

package/node_modules/@luxusai/pi-hindsight/extensions/lifecycle/retain-receipts.ts

@@ -0,0 +1,132 @@
+import { mkdir, readFile, rename, writeFile } from "node:fs/promises";
+import { dirname, resolve } from "node:path";
+import { redactSecrets } from "../utils/sanitize.js";
+import type { RetainOutcome, UpdateMode } from "../types.js";
+
+export interface RetainReceipt {
+  createdAt: string;
+  bankId: string;
+  documentId: string;
+  queueJobId: string;
+  updateMode: UpdateMode;
+  source: "tool" | "import" | "queue";
+  context: string;
+  tags: string[];
+  outcome?: RetainOutcome;
+}
+
+export interface RetainReceiptHistory {
+  version: 1;
+  receipts: RetainReceipt[];
+}
+
+const MAX_RECEIPTS = 50;
+const MAX_RECEIPT_CONTEXT_CHARS = 1000;
+
+export function retainReceiptsPath(cwd: string): string {
+  return resolve(cwd, ".pi/hindsight/retain-receipts.json");
+}
+
+function emptyHistory(): RetainReceiptHistory {
+  return { version: 1, receipts: [] };
+}
+
+export async function readRetainReceipts(path: string): Promise<RetainReceiptHistory> {
+  try {
+    const parsed = JSON.parse(await readFile(path, "utf8")) as Partial<RetainReceiptHistory>;
+    if (parsed.version !== 1 || !Array.isArray(parsed.receipts)) return emptyHistory();
+    return { version: 1, receipts: parsed.receipts.filter(isRetainReceipt).slice(0, MAX_RECEIPTS) };
+  } catch (error) {
+    if ((error as NodeJS.ErrnoException).code === "ENOENT") return emptyHistory();
+    return emptyHistory();
+  }
+}
+
+function isRetainReceipt(value: unknown): value is RetainReceipt {
+  if (!value || typeof value !== "object") return false;
+  const receipt = value as Record<string, unknown>;
+  return (
+    typeof receipt.createdAt === "string" &&
+    typeof receipt.bankId === "string" &&
+    typeof receipt.documentId === "string" &&
+    typeof receipt.queueJobId === "string" &&
+    (receipt.updateMode === "append" || receipt.updateMode === "replace") &&
+    (receipt.source === "tool" || receipt.source === "import" || receipt.source === "queue") &&
+    typeof receipt.context === "string" &&
+    Array.isArray(receipt.tags) &&
+    receipt.tags.every((tag) => typeof tag === "string")
+  );
+}
+
+type ReceiptInput = Omit<RetainReceipt, "createdAt"> & { createdAt?: string };
+type ReceiptOptions = { redactSecrets?: boolean; maxContextChars?: number };
+
+function receiptContext(context: string, options: ReceiptOptions): string {
+  const redacted = options.redactSecrets === false ? context : redactSecrets(context);
+  const maxChars = options.maxContextChars ?? MAX_RECEIPT_CONTEXT_CHARS;
+  if (redacted.length <= maxChars) return redacted;
+  return `${redacted.slice(0, maxChars)}…[truncated]`;
+}
+
+function buildReceipt(receipt: ReceiptInput, options: ReceiptOptions): RetainReceipt {
+  return {
+    createdAt: receipt.createdAt ?? new Date().toISOString(),
+    bankId: receipt.bankId,
+    documentId: receipt.documentId,
+    queueJobId: receipt.queueJobId,
+    updateMode: receipt.updateMode,
+    source: receipt.source,
+    context: receiptContext(receipt.context, options),
+    tags: receipt.tags,
+    ...(receipt.outcome ? { outcome: receipt.outcome } : {}),
+  };
+}
+
+function mergeReceipt(receipts: RetainReceipt[], next: RetainReceipt): RetainReceipt[] {
+  return [
+    next,
+    ...receipts.filter(
+      (existing) => existing.documentId !== next.documentId || existing.bankId !== next.bankId,
+    ),
+  ];
+}
+
+async function writeReceiptHistory(
+  cwd: string,
+  receipts: RetainReceipt[],
+): Promise<RetainReceiptHistory> {
+  const path = retainReceiptsPath(cwd);
+  const history = { version: 1 as const, receipts: receipts.slice(0, MAX_RECEIPTS) };
+  await mkdir(dirname(path), { recursive: true });
+  const tempPath = `${path}.${process.pid}.${Date.now()}.tmp`;
+  await writeFile(tempPath, `${JSON.stringify(history, null, 2)}\n`, "utf8");
+  await rename(tempPath, path);
+  return history;
+}
+
+export async function appendRetainReceipt(
+  cwd: string,
+  receipt: ReceiptInput,
+  options: ReceiptOptions = {},
+): Promise<RetainReceiptHistory> {
+  const current = await readRetainReceipts(retainReceiptsPath(cwd));
+  return writeReceiptHistory(cwd, mergeReceipt(current.receipts, buildReceipt(receipt, options)));
+}
+
+export async function appendRetainReceipts(
+  cwd: string,
+  receiptInputs: ReceiptInput[],
+  options: ReceiptOptions = {},
+): Promise<RetainReceiptHistory> {
+  if (receiptInputs.length === 0) return readRetainReceipts(retainReceiptsPath(cwd));
+  const current = await readRetainReceipts(retainReceiptsPath(cwd));
+  let receipts = current.receipts;
+  for (const input of receiptInputs)
+    receipts = mergeReceipt(receipts, buildReceipt(input, options));
+  return writeReceiptHistory(cwd, receipts);
+}
+
+export async function listRetainReceipts(cwd: string, limit = 10): Promise<RetainReceipt[]> {
+  const history = await readRetainReceipts(retainReceiptsPath(cwd));
+  return history.receipts.slice(0, Math.max(0, limit));
+}

package/node_modules/@luxusai/pi-hindsight/extensions/lifecycle/retain.ts

@@ -0,0 +1,228 @@
+import type { AgentEndEvent } from "@earendil-works/pi-coding-agent";
+import type {
+  HindsightLikeClient,
+  HindsightObservationScopes,
+  ResolvedConfig,
+  RetainJob,
+  RetainOutcome,
+  UpdateMode,
+} from "../types.js";
+import { baseTags } from "../banks/banking.js";
+import { projectMessages } from "../utils/messages.js";
+import { redactError } from "../utils/sanitize.js";
+import { contextLabel, liveDocumentId, stableSessionId } from "../utils/session.js";
+import {
+  enqueueRetain,
+  enqueueRetainCoalesced,
+  flushRetain,
+  summarizeRetain,
+  summarizeRetainQueue,
+  retainQueuePath,
+  type FlushRetainQueueResult,
+} from "../queue/queue.js";
+import { createMemoryIdentity } from "../operations/memory-identity.js";
+import { LIVE_SESSION_RETAIN_STRATEGY } from "../banks/retain-strategies.js";
+import { expandObservationScopes } from "./observation-scopes.js";
+import { buildRetainJob as buildRetainJobCore } from "./retain-job-builder.js";
+import { appendRetainReceipts } from "./retain-receipts.js";
+
+function hasRetainOutcome(outcome: { itemsCount?: number; operations?: number; tokens?: number }) {
+  return (
+    outcome.itemsCount !== undefined ||
+    outcome.operations !== undefined ||
+    outcome.tokens !== undefined
+  );
+}
+
+export async function recordRetainDeliveries(
+  cwd: string,
+  config: ResolvedConfig,
+  result: FlushRetainQueueResult,
+): Promise<void> {
+  if (!result.delivered?.length) return;
+  await appendRetainReceipts(
+    cwd,
+    result.delivered.map((delivery) => ({
+      queueJobId: delivery.queueJobId,
+      bankId: delivery.bankId,
+      documentId: delivery.documentId,
+      updateMode: delivery.updateMode,
+      source: "queue" as const,
+      context: delivery.context,
+      tags: delivery.tags,
+      ...(hasRetainOutcome(delivery.outcome) ? { outcome: delivery.outcome } : {}),
+    })),
+    { redactSecrets: config.retain.redactSecrets },
+  );
+}
+
+export function buildRetainJob(args: {
+  config: ResolvedConfig;
+  cwd: string;
+  sessionFile?: string;
+  bankId: string;
+  messages: AgentEndEvent["messages"];
+  extraTags?: string[];
+}): RetainJob | undefined {
+  const projected = projectMessages(args.messages, args.config);
+  if (projected.length === 0) return undefined;
+  const content = JSON.stringify(projected, null, 2);
+  const sessionId = stableSessionId(args.sessionFile, args.cwd);
+  const identity = createMemoryIdentity(args.cwd, args.config, args.sessionFile);
+  const observationScopes = args.config.observations.enabled
+    ? expandObservationScopes(args.config.observations.scopes, {
+        ...identity,
+        projectBankId: args.bankId,
+      })
+    : [];
+  return buildRetainJobCore({
+    config: args.config,
+    bankId: args.bankId,
+    content,
+    context: contextLabel(args.cwd, args.sessionFile, args.config.agentUse),
+    documentId: liveDocumentId(args.sessionFile, args.cwd),
+    updateMode: args.config.retain.updateMode,
+    tags: [...new Set([...baseTags(args.cwd, sessionId, args.config), ...(args.extraTags ?? [])])],
+    metadata: {
+      cwd: args.cwd,
+      imported: "false",
+      ...(args.sessionFile ? { pi_session_file: args.sessionFile } : {}),
+    },
+    ...(observationScopes.length ? { observationScopes } : {}),
+    // Coding bank templates define named strategies; live sessions use conversation.
+    strategy: LIVE_SESSION_RETAIN_STRATEGY,
+  });
+}
+
+export async function enqueueRetainFromAgentEnd(args: {
+  event: AgentEndEvent;
+  cwd: string;
+  sessionFile?: string;
+  config: ResolvedConfig;
+  client: HindsightLikeClient;
+  bankId: string;
+  extraTags?: string[];
+}): Promise<{ queued: boolean; sent: number; remaining: number; reflectError?: string }> {
+  if (!args.config.enabled || !args.config.retain.enabled)
+    return { queued: false, sent: 0, remaining: 0 };
+  const job = buildRetainJob({
+    config: args.config,
+    cwd: args.cwd,
+    ...(args.sessionFile ? { sessionFile: args.sessionFile } : {}),
+    bankId: args.bankId,
+    messages: args.event.messages,
+    ...(args.extraTags ? { extraTags: args.extraTags } : {}),
+  });
+  if (!job) return { queued: false, sent: 0, remaining: 0 };
+
+  // Coalesced delivery: merge this delta into any compatible pending job and defer the
+  // remote flush to the session boundary (shutdown) / periodic flush. This collapses the
+  // per-agent_end write storm into a few larger operations. Durability is unchanged: the
+  // job is persisted to the on-disk queue before we return.
+  if (args.config.retain.delivery === "coalesced") {
+    await enqueueRetainCoalesced(args.cwd, args.config, job);
+    const summary = await summarizeRetainQueue(retainQueuePath(args.cwd, args.config));
+    return { queued: true, sent: 0, remaining: summary.active.valid };
+  }
+
+  await enqueueRetain(args.cwd, args.config, job);
+  const result = await flushRetain(args.cwd, args.config, args.client);
+  await recordRetainDeliveries(args.cwd, args.config, result);
+  let reflectError: string | undefined;
+  if (args.config.retain.postRetainReflect) {
+    try {
+      await args.client.reflect(
+        args.bankId,
+        "Reflect on the recently retained session to extract insights",
+        { context: "Post-retain reflection" },
+      );
+    } catch (error) {
+      // Best-effort reflect after retain; the retain itself already succeeded above.
+      // Surface the (redacted) cause to the caller instead of swallowing it silently,
+      // so callers with a notification channel can make it debug-visible.
+      reflectError = redactError(error);
+    }
+  }
+  return {
+    queued: true,
+    sent: result.sent,
+    remaining: result.remaining,
+    ...(reflectError ? { reflectError } : {}),
+  };
+}
+
+export type DurableRetainSource = "auto" | "tool" | "command" | "import";
+
+export interface RetainDurablyArgs {
+  cwd: string;
+  config: ResolvedConfig;
+  client: HindsightLikeClient;
+  bankId: string;
+  content: string;
+  context: string;
+  tags: string[];
+  documentId: string;
+  updateMode: UpdateMode;
+  metadata?: Record<string, string>;
+  source: DurableRetainSource;
+  timestamp?: string;
+  observationScopes?: HindsightObservationScopes;
+  documentTags?: string[];
+  entities?: RetainJob["item"]["entities"];
+  async?: boolean;
+}
+
+export interface RetainDurablyResult {
+  enqueued: boolean;
+  sent: number;
+  remaining: number;
+  deadLettered: number;
+  bankId: string;
+  documentId: string;
+  queueJobId: string;
+  updateMode: UpdateMode;
+  operationIds?: string[];
+  outcome?: RetainOutcome;
+}
+
+export function buildDurableRetainJob(args: Omit<RetainDurablyArgs, "client">): RetainJob {
+  return buildRetainJobCore({
+    ...args,
+    metadata: {
+      ...args.metadata,
+      source: "pi-hindsight",
+      retainSource: args.source,
+    },
+  });
+}
+
+export async function retainDurably(args: RetainDurablyArgs): Promise<RetainDurablyResult> {
+  const job = buildDurableRetainJob(args);
+  const receipt = {
+    bankId: job.bankId,
+    documentId: job.documentId,
+    queueJobId: job.id,
+    updateMode: job.updateMode,
+  };
+  const enqueueResult = await enqueueRetain(args.cwd, args.config, job);
+  if (enqueueResult.previousLength > 0) {
+    const summary = await summarizeRetain(args.cwd, args.config);
+    return {
+      ...receipt,
+      enqueued: true,
+      sent: 0,
+      remaining: summary.active.valid + summary.active.malformed,
+      deadLettered: 0,
+    };
+  }
+  const result = await flushRetain(args.cwd, args.config, args.client, { maxJobs: 1 });
+  return {
+    ...receipt,
+    enqueued: true,
+    sent: result.sent,
+    remaining: result.remaining,
+    deadLettered: result.deadLettered,
+    ...(result.operationIds ? { operationIds: result.operationIds } : {}),
+    ...(result.outcome ? { outcome: result.outcome } : {}),
+  };
+}

package/node_modules/@luxusai/pi-hindsight/extensions/operations/memory-bank-template-operations.ts

@@ -0,0 +1,74 @@
+import {
+  getBuiltInBankTemplate,
+  listBankTemplatesForAgentUse,
+  listBuiltInBankTemplates,
+  resolveBankTemplateManifest,
+} from "../banks/bank-templates.js";
+import { resolveProjectIdentity } from "../banks/banking.js";
+import { resolveOperationBank } from "../banks/bank-selection.js";
+import { seedKnowledgePages } from "../banks/knowledge-page-seed.js";
+import type { MemoryOperationsDeps } from "./memory-operation-types.js";
+
+export function createBankTemplateOperations(deps: MemoryOperationsDeps) {
+  return {
+    listBankTemplates() {
+      return listBuiltInBankTemplates();
+    },
+
+    listBankTemplatesForAgentUse() {
+      return listBankTemplatesForAgentUse(deps.getConfig().agentUse);
+    },
+
+    async applyBankTemplate(args: { templateId: string; dryRun?: boolean }) {
+      const template = getBuiltInBankTemplate(args.templateId);
+      if (!template) {
+        throw new Error(
+          `Unknown bank template id: ${args.templateId}. Open /hindsight and press t to list sets for the current agent use.`,
+        );
+      }
+      const client = deps.getClient();
+      if (!client.importBankTemplate) {
+        throw new Error("Hindsight client does not support bank template import.");
+      }
+      const config = deps.getConfig();
+      // Each bundled template already declares which bank kind it targets, so apply resolves
+      // that bank directly instead of taking a separate --bank override.
+      const bankId = resolveOperationBank({
+        requestedBank: template.target === "user" ? "global" : undefined,
+        config,
+        projectBankId: deps.getProjectBankId(),
+      });
+      const bankMissionSettings =
+        template.target === "user" ? config.banks.user : config.banks.project;
+      const cwd = (deps as { getCwd?: () => string }).getCwd?.() ?? process.cwd();
+      const resolvedProjectId =
+        template.target === "project" ? resolveProjectIdentity(cwd, config).projectId : undefined;
+      const manifest = resolveBankTemplateManifest(template, bankMissionSettings, {
+        ...(resolvedProjectId ? { projectId: resolvedProjectId } : {}),
+      });
+      const dryRun = args.dryRun ?? true;
+      const result = await client.importBankTemplate(bankId, manifest, { dryRun });
+      // Coding project templates also seed the knowledge-page taxonomy when the server supports pages.
+      let knowledgeSeed: Awaited<ReturnType<typeof seedKnowledgePages>> | undefined;
+      if (template.id === "pi-coding-project") {
+        const baseTags = [
+          "source:pi",
+          ...(resolvedProjectId ? [`project:${resolvedProjectId}`] : []),
+        ];
+        knowledgeSeed = await seedKnowledgePages({
+          client,
+          bankId,
+          baseTags,
+          dryRun,
+        });
+      }
+      return {
+        bankId,
+        template,
+        dryRun,
+        result,
+        ...(knowledgeSeed ? { knowledgeSeed } : {}),
+      };
+    },
+  };
+}

package/node_modules/@luxusai/pi-hindsight/extensions/operations/memory-config-operations.ts

@@ -0,0 +1,16 @@
+import type { ConfigureMemoryArgs, MemoryOperationsDeps } from "./memory-operation-types.js";
+import { configureMemory, initMemoryConfig } from "../config/config-writer.js";
+
+export function createConfigOperations(deps: MemoryOperationsDeps) {
+  return {
+    async configure(cwd: string, args: ConfigureMemoryArgs) {
+      return configureMemory(cwd, args, deps);
+    },
+
+    async init(cwd: string) {
+      const result = await initMemoryConfig(cwd, deps);
+      deps.reloadConfig?.(cwd);
+      return result;
+    },
+  };
+}