@etiquekit/etq 1.0.12 → 1.0.14

package/packages/control/src/memory/drain-import.ts

@@ -0,0 +1,207 @@
+import { createHash } from 'node:crypto';
+import { mkdir, writeFile } from 'node:fs/promises';
+import { dirname, posix, resolve } from 'node:path';
+import { stringify as stringifyYaml } from 'yaml';
+import { assertSharedMemoryScopeId, assertSharedMemoryText } from './sanitize';
+import {
+  addMemoryStoreCandidate,
+  listMemoryStoreCandidateEntries,
+  loadMemoryStoreManifest,
+  readMemoryStoreCandidateContent,
+  type MemoryStoreEntry,
+} from './store-lifecycle';
+import type { EvidenceRef } from './types';
+
+export interface MemoryDrainImportReceipt {
+  schema: 'memory-drain-import-receipt.v0';
+  receipt_id: string;
+  created_at: string;
+  source: {
+    workspace_ref: string;
+    store_id: string;
+    candidate_count: number;
+  };
+  target: {
+    workspace_ref: string;
+    store_id: string;
+    imported_count: number;
+  };
+  imported_entries: Array<{
+    source_entry_id: string;
+    source_content_ref: string;
+    target_entry_id: string;
+    target_content_ref: string;
+    target_path: string;
+    evidence_refs: EvidenceRef[];
+  }>;
+  authority_boundary: {
+    drain_can_authorize_work: false;
+    drain_can_promote_memory: false;
+    drain_can_write_parent_ledger: false;
+    runtime_pod_can_write_parent_ledger: false;
+    imported_entries_are_candidates: true;
+    source_store_mutated: false;
+  };
+}
+
+export interface DrainImportMemoryCandidatesOptions {
+  sourceProjectRoot: string;
+  targetProjectRoot: string;
+  sourceStoresRoot?: string;
+  targetStoresRoot?: string;
+  sourceStoreId: string;
+  targetStoreId: string;
+  sourceWorkspaceRef?: string;
+  targetWorkspaceRef?: string;
+  pathPrefix?: string;
+  receiptPath?: string;
+  at?: string;
+}
+
+function isoNow(at?: string): string {
+  const value = at ?? new Date().toISOString();
+  if (!Number.isFinite(Date.parse(value))) throw new Error('--at must be an ISO-8601 timestamp');
+  return new Date(value).toISOString();
+}
+
+function stableHash(value: unknown): string {
+  return createHash('sha256').update(JSON.stringify(value)).digest('hex').slice(0, 16);
+}
+
+function assertOpaque(value: string, field: string): string {
+  assertSharedMemoryText(value, field);
+  if (value.startsWith('/') || value.startsWith('\\') || /^[A-Za-z]:[\\/]/.test(value)) {
+    throw new Error(`${field} must be opaque or repo-relative, not a local absolute path`);
+  }
+  return value;
+}
+
+function assertStoreId(value: string, field: string): string {
+  assertSharedMemoryScopeId(value, field);
+  return value;
+}
+
+function assertPathPrefix(raw: string | undefined, sourceStoreId: string): string {
+  const value = raw ?? `imports/${sourceStoreId}`;
+  if (value.startsWith('/') || value.startsWith('\\') || /^[A-Za-z]:[\\/]/.test(value)) {
+    throw new Error('--path-prefix must be repo-relative');
+  }
+  if (value.includes('\\')) throw new Error('--path-prefix must use POSIX / separators');
+  const normalized = posix.normalize(value);
+  if (normalized === '.' || normalized === '..' || normalized.startsWith('../')) {
+    throw new Error('--path-prefix must stay inside the target store');
+  }
+  assertSharedMemoryText(normalized, 'path_prefix');
+  return normalized.replace(/\/+$/g, '');
+}
+
+function evidenceFor(entry: MemoryStoreEntry): EvidenceRef[] {
+  const refs: EvidenceRef[] = [...entry.evidence_refs, { type: 'doc', ref: `memory-entry:${entry.entry_id}` }];
+  const seen = new Set<string>();
+  return refs.filter((ref) => {
+    const key = `${ref.type}:${ref.ref}`;
+    if (seen.has(key)) return false;
+    seen.add(key);
+    return true;
+  });
+}
+
+export async function drainImportMemoryCandidates(
+  options: DrainImportMemoryCandidatesOptions,
+): Promise<MemoryDrainImportReceipt> {
+  const createdAt = isoNow(options.at);
+  const sourceProjectRoot = resolve(options.sourceProjectRoot);
+  const targetProjectRoot = resolve(options.targetProjectRoot);
+  const sourceStoreId = assertStoreId(options.sourceStoreId, 'source_store');
+  const targetStoreId = assertStoreId(options.targetStoreId, 'target_store');
+  const sourceWorkspaceRef = assertOpaque(options.sourceWorkspaceRef ?? 'child-workspace', 'source_workspace_ref');
+  const targetWorkspaceRef = assertOpaque(options.targetWorkspaceRef ?? 'parent-workspace', 'target_workspace_ref');
+  const pathPrefix = assertPathPrefix(options.pathPrefix, sourceStoreId);
+
+  await loadMemoryStoreManifest({
+    projectRoot: sourceProjectRoot,
+    storesRoot: options.sourceStoresRoot,
+    storeId: sourceStoreId,
+  });
+  const targetStore = await loadMemoryStoreManifest({
+    projectRoot: targetProjectRoot,
+    storesRoot: options.targetStoresRoot,
+    storeId: targetStoreId,
+  });
+  if (targetStore.access !== 'read_write' || targetStore.write_policy === 'none') {
+    throw new Error(`target memory store ${targetStoreId} must be read_write`);
+  }
+
+  const sourceEntries = await listMemoryStoreCandidateEntries({
+    projectRoot: sourceProjectRoot,
+    storesRoot: options.sourceStoresRoot,
+    storeId: sourceStoreId,
+  });
+  const imported: MemoryDrainImportReceipt['imported_entries'] = [];
+
+  for (const entry of sourceEntries) {
+    const content = await readMemoryStoreCandidateContent({
+      projectRoot: sourceProjectRoot,
+      storesRoot: options.sourceStoresRoot,
+      storeId: sourceStoreId,
+      entry,
+    });
+    const evidenceRefs = evidenceFor(entry);
+    const targetEntry = await addMemoryStoreCandidate({
+      projectRoot: targetProjectRoot,
+      storesRoot: options.targetStoresRoot,
+      storeId: targetStoreId,
+      path: `${pathPrefix}/${entry.path}`,
+      content,
+      evidenceRefs,
+      at: createdAt,
+    });
+    imported.push({
+      source_entry_id: entry.entry_id,
+      source_content_ref: entry.content_ref,
+      target_entry_id: targetEntry.entry_id,
+      target_content_ref: targetEntry.content_ref,
+      target_path: targetEntry.path,
+      evidence_refs: evidenceRefs,
+    });
+  }
+
+  const receipt: MemoryDrainImportReceipt = {
+    schema: 'memory-drain-import-receipt.v0',
+    receipt_id: `memory-drain-import:${stableHash({
+      sourceWorkspaceRef,
+      targetWorkspaceRef,
+      sourceStoreId,
+      targetStoreId,
+      imported: imported.map((entry) => `${entry.source_entry_id}->${entry.target_entry_id}`),
+    })}`,
+    created_at: createdAt,
+    source: {
+      workspace_ref: sourceWorkspaceRef,
+      store_id: sourceStoreId,
+      candidate_count: sourceEntries.length,
+    },
+    target: {
+      workspace_ref: targetWorkspaceRef,
+      store_id: targetStoreId,
+      imported_count: imported.length,
+    },
+    imported_entries: imported,
+    authority_boundary: {
+      drain_can_authorize_work: false,
+      drain_can_promote_memory: false,
+      drain_can_write_parent_ledger: false,
+      runtime_pod_can_write_parent_ledger: false,
+      imported_entries_are_candidates: true,
+      source_store_mutated: false,
+    },
+  };
+
+  if (options.receiptPath) {
+    const receiptPath = resolve(options.receiptPath);
+    await mkdir(dirname(receiptPath), { recursive: true });
+    await writeFile(receiptPath, stringifyYaml(receipt));
+  }
+
+  return receipt;
+}

package/packages/control/src/memory/indexer.ts

@@ -0,0 +1,284 @@
+import { createHash } from 'node:crypto';
+import { existsSync, statSync } from 'node:fs';
+import { readdir, readFile, writeFile, mkdir } from 'node:fs/promises';
+import { homedir } from 'node:os';
+import { basename, dirname, extname, join, relative, resolve } from 'node:path';
+import {
+  assertCapsuleIsSharedSafe,
+  assertSharedMemoryScopeId,
+  detectMemoryRiskFlags,
+  validateCandidateForPromotion,
+} from './sanitize';
+import { openMemoryStore } from './store';
+import type {
+  EvidenceRef,
+  MemoryCandidate,
+  MemoryCategory,
+  MemoryIndexResult,
+  MemoryRiskFlag,
+  RecallCapsule,
+} from './types';
+
+export const APPROVED_SCAN_DIRS = [
+  'docs/work/ledger',
+  'docs/work/tasks',
+  'docs/work/playbacks',
+  'internal/docs/playbacks',
+  'internal/docs/contracts',
+  '.etiquette/events',
+];
+
+const TEXT_EXTENSIONS = new Set(['.md', '.yaml', '.yml', '.json', '.jsonl', '.ndjson', '.txt']);
+const LOCAL_SEARCH_TEXT_MAX = 8000;
+
+interface IndexedMemoryCandidate {
+  candidate: MemoryCandidate;
+  searchText: string;
+}
+
+export function defaultMemoryDbPath(projectRoot: string): string {
+  return join(projectRoot, '.etiquette', 'state', 'memory', 'index.db');
+}
+
+function safePathSegment(value: string): string {
+  const cleaned = value.replace(/[^A-Za-z0-9_.:-]+/g, '-').replace(/^-+|-+$/g, '');
+  return cleaned || 'workspace';
+}
+
+export function defaultMemoryExportDir(projectRoot: string, workspaceId?: string, now = new Date()): string {
+  const workspace = safePathSegment(workspaceId ?? basename(resolve(projectRoot)));
+  const day = now.toISOString().slice(0, 10);
+  return join(homedir(), '.etiquette', 'exports', workspace, day);
+}
+
+function sha(input: string): string {
+  return createHash('sha256').update(input).digest('hex').slice(0, 16);
+}
+
+function compactWhitespace(value: string): string {
+  return value.replace(/\s+/g, ' ').trim();
+}
+
+function cleanSummary(value: string): string {
+  const compact = compactWhitespace(value.replace(/[`*_>#-]/g, ' '));
+  return compact.length > 260 ? `${compact.slice(0, 257)}...` : compact;
+}
+
+function firstHeading(content: string, fallback: string): string {
+  const heading = content.split('\n').find((line) => line.startsWith('# '));
+  if (!heading) return fallback;
+  return compactWhitespace(heading.replace(/^#+\s*/, ''));
+}
+
+function firstParagraph(content: string, fallback: string): string {
+  const paragraph = content
+    .split(/\n\s*\n/)
+    .map((part) => part.trim())
+    .find((part) => part.length > 0 && !part.startsWith('#') && !part.startsWith('```'));
+  return cleanSummary(paragraph ?? fallback);
+}
+
+function sanitizeLocalSearchText(content: string): string {
+  const kept: string[] = [];
+  for (const line of content.split('\n')) {
+    const withoutFenceMarker = line.replace(/```[A-Za-z0-9_-]*/g, ' ');
+    const restricted = detectMemoryRiskFlags(withoutFenceMarker).filter((flag) => flag !== 'contains_source_snippet');
+    if (restricted.length > 0) continue;
+    const normalized = withoutFenceMarker.replace(/[`*_>#|\[\]{}()]/g, ' ');
+    const compact = compactWhitespace(normalized);
+    if (compact.length > 0) kept.push(compact);
+  }
+  const searchText = compactWhitespace(kept.join(' '));
+  return searchText.length > LOCAL_SEARCH_TEXT_MAX ? searchText.slice(0, LOCAL_SEARCH_TEXT_MAX) : searchText;
+}
+
+function categoryForPath(relPath: string): MemoryCategory {
+  if (relPath.includes('/playbacks/')) return 'playback_candidate';
+  if (relPath.includes('/ledger/') || relPath.includes('/tasks/')) return 'pattern';
+  if (relPath.includes('hosted-pass-plane') || relPath.includes('event-spine')) return 'decision';
+  if (relPath.includes('workflow-memory')) return 'adapter_note';
+  return 'onboarding';
+}
+
+function evidenceForPath(relPath: string): EvidenceRef {
+  if (relPath.includes('/ledger/')) return { type: 'ledger', ref: relPath };
+  if (relPath.includes('/playbacks/')) return { type: 'playback', ref: relPath };
+  if (relPath.includes('.etiquette/events/')) return { type: 'event', ref: relPath };
+  return { type: 'doc', ref: relPath };
+}
+
+async function collectFiles(root: string, relDir: string): Promise<string[]> {
+  const absDir = join(root, relDir);
+  if (!existsSync(absDir)) return [];
+  const out: string[] = [];
+  async function walk(dir: string): Promise<void> {
+    const entries = await readdir(dir, { withFileTypes: true });
+    for (const entry of entries) {
+      const abs = join(dir, entry.name);
+      if (entry.isDirectory()) {
+        if (entry.name === 'node_modules' || entry.name === '.git' || entry.name === '_worktrees') continue;
+        await walk(abs);
+        continue;
+      }
+      if (!entry.isFile()) continue;
+      if (!TEXT_EXTENSIONS.has(extname(entry.name))) continue;
+      out.push(relative(root, abs));
+    }
+  }
+  await walk(absDir);
+  return out.sort();
+}
+
+export async function collectApprovedMemoryFiles(projectRoot: string): Promise<string[]> {
+  const batches = await Promise.all(APPROVED_SCAN_DIRS.map((dir) => collectFiles(projectRoot, dir)));
+  return batches.flat().sort();
+}
+
+export async function candidateFromFile(projectRoot: string, relPath: string): Promise<MemoryCandidate> {
+  return (await indexedCandidateFromFile(projectRoot, relPath)).candidate;
+}
+
+async function indexedCandidateFromFile(projectRoot: string, relPath: string): Promise<IndexedMemoryCandidate> {
+  const abs = join(projectRoot, relPath);
+  const content = await readFile(abs, 'utf-8');
+  const fallbackTitle = basename(relPath, extname(relPath)).replace(/[-_]+/g, ' ');
+  const title = firstHeading(content, fallbackTitle);
+  const summary = firstParagraph(content, title);
+  const riskFlags = new Set<MemoryRiskFlag>();
+  for (const flag of detectMemoryRiskFlags(`${title}\n${summary}`)) {
+    riskFlags.add(flag);
+  }
+  const stat = statSync(abs);
+  return {
+    candidate: {
+      candidate_id: `cand_${sha(`${relPath}:${stat.mtimeMs}:${stat.size}`)}`,
+      adapter: 'local_importer',
+      query: relPath,
+      category: categoryForPath(relPath),
+      title,
+      summary,
+      source_refs: [evidenceForPath(relPath)],
+      confidence: 'medium',
+      risk_flags: [...riskFlags],
+    },
+    searchText: sanitizeLocalSearchText(content),
+  };
+}
+
+export function promoteCandidate(
+  candidate: MemoryCandidate,
+  opts: { localPrivate?: boolean; now?: string; scope?: RecallCapsule['scope'] } = {},
+): RecallCapsule | null {
+  const flags = validateCandidateForPromotion(candidate, opts);
+  const blocking = flags.filter((flag) => flag !== 'stale' && flag !== 'contradicts_current_ledger');
+  if (blocking.length > 0) return null;
+  const now = opts.now ?? new Date().toISOString();
+  const stale = flags.includes('stale') || flags.includes('contradicts_current_ledger');
+  const capsule: RecallCapsule = {
+    schema: 'recall-capsule.v0',
+    capsule_id: `mem_${sha(`${candidate.title}:${candidate.source_refs.map((ref) => ref.ref).join('|')}`)}`,
+    created_at: now,
+    updated_at: now,
+    scope: opts.scope ?? {},
+    category: candidate.category,
+    title: candidate.title,
+    summary: candidate.summary,
+    why_it_matters: `Imported from ${candidate.source_refs[0]?.ref}; use as cited context, not authority.`,
+    evidence_refs: candidate.source_refs,
+    confidence: candidate.confidence,
+    staleness: {
+      status: stale ? 'watch' : 'current',
+      stale_if: stale ? 'current ledger or receipt contradicts this capsule' : undefined,
+    },
+    recommended_action: {
+      type: candidate.category === 'playback_candidate' ? 'run_playback' : 'read_before_work',
+    },
+    authority_boundary: {
+      can_authorize_work: false,
+      can_close_task: false,
+      can_replace_receipt: false,
+    },
+  };
+  assertCapsuleIsSharedSafe(capsule);
+  return capsule;
+}
+
+export async function indexMemory(options: {
+  projectRoot: string;
+  dbPath?: string;
+  exportDir?: string;
+  localPrivate?: boolean;
+  orgId?: string;
+  workspaceId?: string;
+  confirmSharedExport?: boolean;
+}): Promise<MemoryIndexResult> {
+  const projectRoot = resolve(options.projectRoot);
+  const dbPath = options.dbPath ?? defaultMemoryDbPath(projectRoot);
+  const sharedExport = options.exportDir !== undefined;
+  if (sharedExport) {
+    if (options.localPrivate) {
+      throw new Error('--local-private cannot be combined with shared capsule export');
+    }
+    if (!options.confirmSharedExport) {
+      throw new Error('--confirm-shared-export is required when writing shared capsule exports');
+    }
+    if (!options.orgId) {
+      throw new Error('--org-id is required when writing shared capsule exports');
+    }
+    assertSharedMemoryScopeId(options.orgId, '--org-id');
+    if (options.workspaceId) assertSharedMemoryScopeId(options.workspaceId, '--workspace-id');
+  }
+  const store = openMemoryStore(dbPath);
+  try {
+    store.reset();
+    const files = await collectApprovedMemoryFiles(projectRoot);
+    const capsuleIds: string[] = [];
+    let rejected = 0;
+    for (const relPath of files) {
+      const { candidate, searchText } = await indexedCandidateFromFile(projectRoot, relPath);
+      store.insertCandidate(candidate);
+      const flags = validateCandidateForPromotion(candidate, {
+        localPrivate: options.localPrivate,
+      });
+      const blocking = flags.filter((flag) => flag !== 'stale' && flag !== 'contradicts_current_ledger');
+      if (blocking.length > 0) {
+        rejected += 1;
+        store.markCandidate(candidate.candidate_id, 'rejected', blocking);
+        continue;
+      }
+      if (sharedExport && candidate.confidence === 'low') {
+        rejected += 1;
+        store.markCandidate(candidate.candidate_id, 'rejected', ['confidence_low_for_shared_export']);
+        continue;
+      }
+      const capsule = promoteCandidate(candidate, {
+        localPrivate: options.localPrivate,
+        scope: sharedExport ? {
+          org_id: options.orgId,
+          workspace_id: options.workspaceId,
+        } : {},
+      });
+      if (!capsule) {
+        rejected += 1;
+        store.markCandidate(candidate.candidate_id, 'rejected', ['no_durable_evidence']);
+        continue;
+      }
+      store.markCandidate(candidate.candidate_id, 'promoted', []);
+      store.insertCapsule(capsule, searchText);
+      capsuleIds.push(capsule.capsule_id);
+      if (options.exportDir) {
+        const target = join(options.exportDir, `${capsule.capsule_id}.json`);
+        await mkdir(dirname(target), { recursive: true });
+        await writeFile(target, `${JSON.stringify(capsule, null, 2)}\n`, 'utf-8');
+      }
+    }
+    return {
+      candidates: files.length,
+      promoted: capsuleIds.length,
+      rejected,
+      capsuleIds,
+    };
+  } finally {
+    store.close();
+  }
+}

package/packages/control/src/memory/query.ts

@@ -0,0 +1,75 @@
+import { resolve } from 'node:path';
+import { defaultMemoryDbPath } from './indexer';
+import { openMemoryStore } from './store';
+import type { CompactMemoryResult, RecallCapsule, RejectedMemoryCandidate } from './types';
+
+export function searchMemory(options: {
+  projectRoot: string;
+  dbPath?: string;
+  query: string;
+  limit: number;
+}): CompactMemoryResult[] {
+  const projectRoot = resolve(options.projectRoot);
+  const store = openMemoryStore(options.dbPath ?? defaultMemoryDbPath(projectRoot));
+  try {
+    return store.search(options.query, options.limit);
+  } finally {
+    store.close();
+  }
+}
+
+export function showMemory(options: {
+  projectRoot: string;
+  dbPath?: string;
+  capsuleId: string;
+}): RecallCapsule | null {
+  const projectRoot = resolve(options.projectRoot);
+  const store = openMemoryStore(options.dbPath ?? defaultMemoryDbPath(projectRoot));
+  try {
+    return store.show(options.capsuleId);
+  } finally {
+    store.close();
+  }
+}
+
+export function staleMemory(options: {
+  projectRoot: string;
+  dbPath?: string;
+  limit: number;
+}): CompactMemoryResult[] {
+  const projectRoot = resolve(options.projectRoot);
+  const store = openMemoryStore(options.dbPath ?? defaultMemoryDbPath(projectRoot));
+  try {
+    return store.stale(options.limit);
+  } finally {
+    store.close();
+  }
+}
+
+export function rejectedMemory(options: {
+  projectRoot: string;
+  dbPath?: string;
+  limit: number;
+}): RejectedMemoryCandidate[] {
+  const projectRoot = resolve(options.projectRoot);
+  const store = openMemoryStore(options.dbPath ?? defaultMemoryDbPath(projectRoot));
+  try {
+    return store.rejected(options.limit);
+  } finally {
+    store.close();
+  }
+}
+
+export function preflightMemory(options: {
+  projectRoot: string;
+  dbPath?: string;
+  topic: string;
+  limit: number;
+}): CompactMemoryResult[] {
+  return searchMemory({
+    projectRoot: options.projectRoot,
+    dbPath: options.dbPath,
+    query: options.topic,
+    limit: options.limit,
+  });
+}

package/packages/control/src/memory/sanitize.ts

@@ -0,0 +1,50 @@
+import {
+  assertSharedMemoryScopeId,
+  assertSharedMemoryText,
+  detectMemoryRiskFlags,
+} from '../shared/sanitize';
+import type { MemoryCandidate, MemoryRiskFlag, RecallCapsule } from './types';
+
+export {
+  assertSharedMemoryScopeId,
+  assertSharedMemoryText,
+  detectMemoryRiskFlags,
+} from '../shared/sanitize';
+
+export function validateEvidenceRefs(candidate: MemoryCandidate): MemoryRiskFlag[] {
+  if (candidate.source_refs.length === 0) return ['no_durable_evidence'];
+  const badRefs = candidate.source_refs.flatMap((ref) => detectMemoryRiskFlags(ref.ref));
+  return badRefs;
+}
+
+export function validateCandidateForPromotion(
+  candidate: MemoryCandidate,
+  opts: { localPrivate?: boolean } = {},
+): MemoryRiskFlag[] {
+  const flags = new Set<MemoryRiskFlag>(candidate.risk_flags);
+  for (const flag of validateEvidenceRefs(candidate)) {
+    flags.add(flag);
+  }
+  for (const value of [candidate.title, candidate.summary, candidate.query]) {
+    for (const flag of detectMemoryRiskFlags(value)) {
+      flags.add(flag);
+    }
+  }
+  if (opts.localPrivate) {
+    flags.delete('contains_private_context');
+  }
+  return [...flags];
+}
+
+export function assertCapsuleIsSharedSafe(capsule: RecallCapsule): void {
+  if (capsule.scope.org_id) assertSharedMemoryScopeId(capsule.scope.org_id, 'scope.org_id');
+  if (capsule.scope.workspace_id) assertSharedMemoryScopeId(capsule.scope.workspace_id, 'scope.workspace_id');
+  if (capsule.scope.repo_ref) assertSharedMemoryText(capsule.scope.repo_ref, 'scope.repo_ref');
+  if (capsule.scope.lane_ref) assertSharedMemoryText(capsule.scope.lane_ref, 'scope.lane_ref');
+  assertSharedMemoryText(capsule.title, 'title');
+  assertSharedMemoryText(capsule.summary, 'summary');
+  assertSharedMemoryText(capsule.why_it_matters, 'why_it_matters');
+  for (const evidence of capsule.evidence_refs) {
+    assertSharedMemoryText(evidence.ref, `evidence_ref:${evidence.type}`);
+  }
+}