@graphorin/memory 0.6.0 → 0.7.0

package/src/context-engine/locale-packs/types.ts

@@ -0,0 +1,177 @@
+/**
+ * Public types for the per-locale assets the
+ * {@link ContextEngine} consumes when assembling the layered system
+ * prompt. The framework ships an English default ({@link enLocalePack});
+ * additional locales are supplied by application code via
+ * {@link defineContextLocalePack}.
+ *
+ * The framework is locale-agnostic - no language is privileged in
+ * core. Every consumer-facing string is reachable through this
+ * surface so consumers can swap them without forking the framework.
+ *
+ * @packageDocumentation
+ */
+
+/**
+ * Trigger mode used by the auto-recall heuristic. The framework ships
+ * regex patterns for the bundled locale; consumers may extend or
+ * replace them via {@link defineContextLocalePack}.
+ *
+ * @stable
+ */
+export interface AutoRecallTriggers {
+  /**
+   * Case-insensitive regex set evaluated against the last user
+   * message. A match indicates the model would benefit from the
+   * top-K facts being injected into Layer 6.
+   */
+  readonly factTriggers: ReadonlyArray<RegExp>;
+}
+
+/**
+ * Layer 1 template fragments. The `composeTemplate` function
+ * picks the right fragment based on `memoryBaseMode`.
+ *
+ * @stable
+ */
+export interface BaseTemplateFragments {
+  /** Full educational template (~250-350 tokens). */
+  readonly full: string;
+  /** Minimal template for high-end models (~80-120 tokens). */
+  readonly minimal: string;
+}
+
+/**
+ * D4 inbound-sanitization preamble fragment. Injected after the
+ * cache breakpoint (Layer 5/6 territory) on steps containing
+ * untrusted tool output.
+ *
+ * @stable
+ */
+export interface InboundSanitizationPreamble {
+  /** Verbatim text appended to the system message when fired. */
+  readonly text: string;
+}
+
+/**
+ * Structured section-template fragments used by the auto-compaction
+ * summarizer (RB-46). Section order is fixed; only the per-section
+ * header / preamble text is locale-extensible.
+ *
+ * Each entry is the human-readable header for the corresponding
+ * section. Section indices are 1-based to match the documented
+ * layout in the architecture doc.
+ *
+ * @stable
+ */
+export interface CompactionSummaryTemplate {
+  /** Preamble injected at the top of the summarizer prompt. */
+  readonly preamble: string;
+  /** 12 section headers. The last two are filled by the harness. */
+  readonly sections: readonly [
+    string, // 1: Session goal and current task
+    string, // 2: Decisions made and rationale
+    string, // 3: Constraints and non-negotiables (C4)
+    string, // 4: Key facts established
+    string, // 5: Open questions and ambiguities
+    string, // 6: Tools used and their outcomes
+    string, // 7: Files / artifacts referenced
+    string, // 8: Persona / preferences / project rules surfaced
+    string, // 9: Errors encountered and resolutions (SOTA-6)
+    string, // 10: Next steps (SOTA-6)
+    string, // 11: Recent turns preserved verbatim (harness-filled)
+    string, // 12: Compaction metadata (harness-filled)
+  ];
+}
+
+/**
+ * A `LocalePack` collects every consumer-facing string the
+ * ContextEngine renders. Consumers register additional locales via
+ * {@link defineContextLocalePack}. The framework is locale-agnostic
+ * - no language is privileged in core.
+ *
+ * @stable
+ */
+export interface ContextLocalePack {
+  /** Stable lowercase identifier (`'en'`, `'de'`, `'fr'`, …). */
+  readonly id: string;
+  /** Layer 1 base template fragments. */
+  readonly baseTemplate: BaseTemplateFragments;
+  /** Auto-recall trigger regex set. */
+  readonly autoRecallTriggers: AutoRecallTriggers;
+  /** Inbound-sanitization preamble (D4 - RB-43). */
+  readonly inboundSanitizationPreamble: InboundSanitizationPreamble;
+  /** Structured compaction summary template (RB-46). */
+  readonly compactionSummaryTemplate: CompactionSummaryTemplate;
+}
+
+/**
+ * Builder used by application code that wants to ship a custom
+ * locale pack. The builder freezes every input so the pack can be
+ * safely reused across multiple `Memory` instances without
+ * accidental mutation.
+ *
+ * Partial packs are accepted: any field omitted falls back to the
+ * English default at compose time, with a one-time WARN per locale
+ * per missing surface (the safety guarantee is preserved even when
+ * an operator installs a partial locale pack).
+ *
+ * @stable
+ */
+export interface PartialContextLocalePack {
+  readonly id: string;
+  readonly baseTemplate?: Partial<BaseTemplateFragments>;
+  readonly autoRecallTriggers?: Partial<AutoRecallTriggers>;
+  readonly inboundSanitizationPreamble?: Partial<InboundSanitizationPreamble>;
+  readonly compactionSummaryTemplate?: Partial<CompactionSummaryTemplate>;
+}
+
+/**
+ * Build a {@link ContextLocalePack} from a partial input. Missing
+ * fields fall back to the English default at compose time.
+ *
+ * @stable
+ */
+export function defineContextLocalePack(input: PartialContextLocalePack): PartialContextLocalePack {
+  if (typeof input.id !== 'string' || input.id.length === 0) {
+    throw new TypeError(
+      '[graphorin/memory] defineContextLocalePack: `id` must be a non-empty lowercase string.',
+    );
+  }
+  return Object.freeze({
+    id: input.id,
+    ...(input.baseTemplate !== undefined
+      ? { baseTemplate: Object.freeze({ ...input.baseTemplate }) }
+      : {}),
+    ...(input.autoRecallTriggers !== undefined
+      ? {
+          autoRecallTriggers: Object.freeze({
+            ...(input.autoRecallTriggers.factTriggers !== undefined
+              ? { factTriggers: Object.freeze([...input.autoRecallTriggers.factTriggers]) }
+              : {}),
+          }) as Partial<AutoRecallTriggers>,
+        }
+      : {}),
+    ...(input.inboundSanitizationPreamble !== undefined
+      ? {
+          inboundSanitizationPreamble: Object.freeze({ ...input.inboundSanitizationPreamble }),
+        }
+      : {}),
+    ...(input.compactionSummaryTemplate !== undefined
+      ? {
+          compactionSummaryTemplate: Object.freeze({
+            ...(input.compactionSummaryTemplate.preamble !== undefined
+              ? { preamble: input.compactionSummaryTemplate.preamble }
+              : {}),
+            ...(input.compactionSummaryTemplate.sections !== undefined
+              ? {
+                  sections: Object.freeze([
+                    ...input.compactionSummaryTemplate.sections,
+                  ]) as CompactionSummaryTemplate['sections'],
+                }
+              : {}),
+          }) as Partial<CompactionSummaryTemplate>,
+        }
+      : {}),
+  });
+}

package/src/context-engine/metadata.ts

@@ -0,0 +1,114 @@
+/**
+ * `gatherMemoryMetadata(...)` - implements the
+ * {@link Memory.metadata} surface for Phase 10d. Aggregates
+ * per-tier counts + last-consolidation timestamps + active
+ * embedder id + active locale.
+ *
+ * @packageDocumentation
+ */
+
+import type { MemoryMetadata, SessionScope } from '@graphorin/core';
+import type { Consolidator } from '../consolidator/index.js';
+import type { MemoryStoreAdapter } from '../internal/storage-adapter.js';
+
+/**
+ * Per-call dependency surface. Mirrors the `Memory` facade fields
+ * the metadata gatherer needs without taking a hard dependency on
+ * the facade type itself (the facade and the gatherer are
+ * mutually-referenced through the engine).
+ *
+ * @stable
+ */
+export interface MemoryMetadataDeps {
+  readonly store: MemoryStoreAdapter;
+  readonly consolidator: Consolidator;
+  readonly embedderId: () => string | null;
+  readonly localeId: string;
+  /** Optional free-form metadata tags surfaced through `MemoryMetadata.tags`. */
+  readonly tags?: ReadonlyArray<string>;
+}
+
+/**
+ * Build the {@link MemoryMetadata} block. Pure async - no side
+ * effects beyond the storage reads.
+ *
+ * @stable
+ */
+export async function gatherMemoryMetadata(
+  scope: SessionScope,
+  deps: MemoryMetadataDeps,
+): Promise<MemoryMetadata> {
+  // CE-5 / MST-6: real `COUNT(*)` surfaces per tier - the old
+  // `search({ query: '*', topK: 1 })` probe was capped at 1 and matched zero
+  // rows on real SQLite (`escapeFtsQuery('*')` tokenises to nothing), so a
+  // production deploy told the model "Facts: 0" regardless of content. Counts
+  // never materialise rows; `messageCount` no longer lists up to 1000 messages.
+  const [blocks, rules, episodeCount, messageCount, factCount, status] = await Promise.all([
+    deps.store.working.list(scope),
+    deps.store.procedural.list(scope),
+    countOrZero(deps.store.episodic, scope),
+    countOrZero(deps.store.session, scope),
+    countOrZero(deps.store.semantic, scope),
+    deps.consolidator.status(),
+  ]);
+  // The metadata block surfaces the embedder id + active locale so
+  // the model can reason about the available embeddings; both are
+  // informational tags exposed through the optional `tags` field.
+  const embedderId = deps.embedderId();
+  const baseTags = deps.tags !== undefined ? [...deps.tags] : [];
+  const tags: string[] = [...baseTags, `locale:${deps.localeId}`];
+  if (embedderId !== null) tags.push(`embedder:${embedderId}`);
+
+  const meta: MemoryMetadata = {
+    factCount,
+    episodeCount,
+    messageCount,
+    // MST-6: "Active rules" excludes still-quarantined (e.g. induced) rules.
+    activeRuleCount: rules.filter((rule) => rule.status !== 'quarantined').length,
+    workingBlockCount: blocks.length,
+    ...(status.lastRunAt !== undefined ? { lastConsolidatedAt: status.lastRunAt } : {}),
+    tags: Object.freeze(tags),
+  };
+  return Object.freeze(meta);
+}
+
+/**
+ * Read a tier's `count(scope)` when the adapter exposes it (CE-5), degrading to
+ * `0` for adapters that do not - honest, and never the old 0/1 probe.
+ */
+async function countOrZero(
+  store: { count?(scope: SessionScope): Promise<number> },
+  scope: SessionScope,
+): Promise<number> {
+  if (typeof store.count !== 'function') return 0;
+  try {
+    return await store.count(scope);
+  } catch {
+    return 0;
+  }
+}
+
+/**
+ * Render the {@link MemoryMetadata} block as the `<memory_metadata>`
+ * XML fragment used inside Layer 5 of the layered system prompt.
+ *
+ * @stable
+ */
+export function renderMetadataBlock(meta: MemoryMetadata): string {
+  const lines = [
+    '<memory_metadata>',
+    `  Working blocks: ${meta.workingBlockCount}`,
+    `  Active rules: ${meta.activeRuleCount}`,
+    `  Indexed messages: ${meta.messageCount}`,
+    `  Episodes: ${meta.episodeCount}`,
+    `  Facts: ${meta.factCount}`,
+  ];
+  if (meta.lastConsolidatedAt !== undefined) {
+    lines.push(`  Last consolidation: ${meta.lastConsolidatedAt}`);
+  }
+  if (meta.tags !== undefined && meta.tags.length > 0) {
+    lines.push(`  Tags: ${meta.tags.join(', ')}`);
+  }
+  lines.push('</memory_metadata>');
+  return lines.join('\n');
+}

package/src/context-engine/preambles/inbound-en.ts

@@ -0,0 +1,19 @@
+/**
+ * Convenience re-export of the English D4 inbound-sanitization
+ * preamble. Lives at the documented path
+ * `@graphorin/memory/context-engine/preambles/inbound-en.ts` so
+ * Phase 12 (agent runtime) can import it directly when wiring
+ * the per-step preamble injection.
+ *
+ * The framework is locale-agnostic - additional preambles plug in
+ * through `defineContextLocalePack({ inboundSanitizationPreamble:
+ * { text } })`. A partial locale pack falls back to this English
+ * default with a one-time WARN per locale.
+ *
+ * @packageDocumentation
+ */
+
+import { enLocalePack } from '../locale-packs/en.js';
+
+/** English D4 preamble text (~80-120 tokens, cache-friendly). */
+export const INBOUND_SANITIZATION_PREAMBLE_EN = enLocalePack.inboundSanitizationPreamble.text;

package/src/context-engine/privacy-filter.ts

@@ -0,0 +1,190 @@
+/**
+ * D2 sensitivity-tier filter. Tags every memory record with a
+ * decision (`'pass'` / `'drop'`) before the assembled prompt is
+ * compiled. The filter is *tier-based* - it trusts the
+ * `record.sensitivity` tag the consolidator / writer set; the
+ * outbound prompt-redaction middleware (D3, ADR-045) is the
+ * universal backstop for content the filter cannot see (raw user
+ * input, tool results, MCP responses, skill bodies, agent
+ * instructions, tool-call args).
+ *
+ * @packageDocumentation
+ */
+
+import type { LocalProviderTrust, Sensitivity } from '@graphorin/core';
+
+/**
+ * Filter input: the record-level sensitivity tag + the per-tier
+ * trust class of the active provider.
+ *
+ * @stable
+ */
+export interface PrivacyFilterContext {
+  /**
+   * Sensitivity tiers the active provider is allowed to receive.
+   * When omitted, the per-tier defaults below apply.
+   */
+  readonly providerAcceptsSensitivity?: ReadonlyArray<Sensitivity>;
+  /**
+   * Trust class of the active provider as classified by
+   * `@graphorin/provider/trust/classify-local-provider.ts`. Defaults
+   * to `'public-tls'` (the conservative cloud default).
+   */
+  readonly providerTrust?: LocalProviderTrust;
+  /**
+   * Per-user opt-in for cloud upload of `'internal'`-tier content.
+   * Default `false` - `'internal'` content does not leave the
+   * loopback boundary unless the user explicitly opts in.
+   */
+  readonly cloudUploadConsent?: boolean;
+  /**
+   * Override the default sensitivity applied to records that are
+   * missing a tag. Default `'internal'` per DEC-126.
+   */
+  readonly defaultSensitivity?: Sensitivity;
+}
+
+/**
+ * Output of {@link decide} for a single record.
+ *
+ * @stable
+ */
+export type PrivacyDecision = 'pass' | 'drop';
+
+/**
+ * Reason the decision was made. Surfaced to the metadata block so
+ * operators can audit per-tier drops.
+ *
+ * @stable
+ */
+export type PrivacyDecisionReason =
+  | 'provider-rejects-secret'
+  | 'provider-rejects-internal'
+  | 'no-cloud-upload-consent'
+  | 'allowed';
+
+/**
+ * Resolve the effective `acceptsSensitivity` set for a provider.
+ * The function is pure: callers can supply a custom matrix to
+ * override the defaults (used by tests + custom adapters).
+ *
+ * Defaults (DEC-149, ADR-013 ext):
+ *
+ * - `'loopback'`  → `['public', 'internal', 'secret']`.
+ * - `'private'`   → `['public', 'internal']`.
+ * - `'public-tls'` / `'public-cleartext'` → `['public']`.
+ *
+ * Per-provider overrides always win over the defaults - pass
+ * `providerAcceptsSensitivity` explicitly to override.
+ *
+ * @stable
+ */
+export function effectiveAcceptsSensitivity(
+  context: PrivacyFilterContext,
+): ReadonlyArray<Sensitivity> {
+  if (context.providerAcceptsSensitivity !== undefined) {
+    return context.providerAcceptsSensitivity;
+  }
+  switch (context.providerTrust ?? 'public-tls') {
+    case 'loopback':
+      return ['public', 'internal', 'secret'];
+    case 'private':
+      return ['public', 'internal'];
+    case 'public-tls':
+    case 'public-cleartext':
+      return ['public'];
+    default:
+      return ['public'];
+  }
+}
+
+/**
+ * Decide whether a single record is safe to send to the active
+ * provider. The decision logic mirrors the DoD spec:
+ *
+ * - `'public'` always passes.
+ * - `'secret'` passes only when the provider explicitly accepts
+ *   `'secret'` AND the trust class is `'loopback'`.
+ * - `'internal'` passes when the provider accepts `'internal'`
+ *   AND `cloudUploadConsent === true` (or the trust class is
+ *   `'loopback'` / `'private'`).
+ *
+ * @stable
+ */
+export function decide(
+  recordSensitivity: Sensitivity | undefined,
+  context: PrivacyFilterContext,
+): { readonly decision: PrivacyDecision; readonly reason: PrivacyDecisionReason } {
+  const sensitivity = recordSensitivity ?? context.defaultSensitivity ?? 'internal';
+  const accepts = effectiveAcceptsSensitivity(context);
+  const trust = context.providerTrust ?? 'public-tls';
+  if (sensitivity === 'public') {
+    return { decision: 'pass', reason: 'allowed' };
+  }
+  if (sensitivity === 'secret') {
+    if (accepts.includes('secret') && trust === 'loopback') {
+      return { decision: 'pass', reason: 'allowed' };
+    }
+    return { decision: 'drop', reason: 'provider-rejects-secret' };
+  }
+  // 'internal'
+  if (!accepts.includes('internal')) {
+    return { decision: 'drop', reason: 'provider-rejects-internal' };
+  }
+  if (trust === 'loopback' || trust === 'private') {
+    return { decision: 'pass', reason: 'allowed' };
+  }
+  if (context.cloudUploadConsent === true) {
+    return { decision: 'pass', reason: 'allowed' };
+  }
+  return { decision: 'drop', reason: 'no-cloud-upload-consent' };
+}
+
+/**
+ * Bookkeeping returned by {@link partition}. Carries both the
+ * surviving + dropped records and a per-reason counter for the
+ * audit trail.
+ *
+ * @stable
+ */
+export interface PartitionResult<TRecord> {
+  readonly kept: ReadonlyArray<TRecord>;
+  readonly dropped: ReadonlyArray<{
+    readonly record: TRecord;
+    readonly reason: PrivacyDecisionReason;
+  }>;
+  readonly counters: Readonly<Record<PrivacyDecisionReason, number>>;
+}
+
+/**
+ * Partition a record list against the supplied filter context.
+ *
+ * @stable
+ */
+export function partition<TRecord extends { readonly sensitivity?: Sensitivity }>(
+  records: ReadonlyArray<TRecord>,
+  context: PrivacyFilterContext,
+): PartitionResult<TRecord> {
+  const kept: TRecord[] = [];
+  const dropped: Array<{ readonly record: TRecord; readonly reason: PrivacyDecisionReason }> = [];
+  const counters: Record<PrivacyDecisionReason, number> = {
+    allowed: 0,
+    'no-cloud-upload-consent': 0,
+    'provider-rejects-internal': 0,
+    'provider-rejects-secret': 0,
+  };
+  for (const record of records) {
+    const outcome = decide(record.sensitivity, context);
+    counters[outcome.reason] += 1;
+    if (outcome.decision === 'pass') {
+      kept.push(record);
+    } else {
+      dropped.push({ record, reason: outcome.reason });
+    }
+  }
+  return Object.freeze({
+    kept: Object.freeze(kept),
+    dropped: Object.freeze(dropped),
+    counters: Object.freeze(counters),
+  });
+}

package/src/context-engine/templates/base-en.ts

@@ -0,0 +1,19 @@
+/**
+ * Convenience re-export of the English Layer 1 base-template
+ * fragments so call sites can import the bundled defaults without
+ * dragging the full locale pack.
+ *
+ * The framework is locale-agnostic - no language is privileged in
+ * core. Application code can register additional locales via
+ * `defineContextLocalePack`.
+ *
+ * @packageDocumentation
+ */
+
+import { enLocalePack } from '../locale-packs/en.js';
+
+/** English Layer 1 in `'full'` mode (~250-350 tokens). */
+export const BASE_TEMPLATE_EN_FULL = enLocalePack.baseTemplate.full;
+
+/** English Layer 1 in `'minimal'` mode (~80-120 tokens). */
+export const BASE_TEMPLATE_EN_MINIMAL = enLocalePack.baseTemplate.minimal;

package/src/context-engine/templates/composer.ts

@@ -0,0 +1,91 @@
+/**
+ * Layer 1 base-template composer. Resolves the locale pack, picks
+ * the right base fragment based on `memoryBaseMode`, and exposes
+ * the per-step inbound preamble as an additive Layer 5/6 fragment
+ * (so the Layer 1-4 cache prefix stays bytes-equal across the run).
+ *
+ * @packageDocumentation
+ */
+
+import type { ContextLocalePack } from '../locale-packs/index.js';
+
+/**
+ * Layer 1 mode discriminator. `'full'` ships the full educational
+ * template (~250-350 tok); `'minimal'` ships the shortened version
+ * (~80-120 tok) for top-tier models.
+ *
+ * @stable
+ */
+export type MemoryBaseMode = 'full' | 'minimal';
+
+/**
+ * Compose Layer 1 from the resolved locale pack.
+ *
+ * @stable
+ */
+export function composeLayer1(pack: ContextLocalePack, mode: MemoryBaseMode): string {
+  return mode === 'minimal' ? pack.baseTemplate.minimal : pack.baseTemplate.full;
+}
+
+/**
+ * Render the optional Layer 2 (`agent_instructions`) wrapper.
+ * Returns the empty string when `instructions` is empty so the
+ * layer is dropped from the assembled prompt.
+ *
+ * @stable
+ */
+export function composeLayer2(instructions: string | undefined): string {
+  if (instructions === undefined || instructions.trim().length === 0) return '';
+  return `<agent_instructions>\n${instructions}\n</agent_instructions>`;
+}
+
+/**
+ * Render Layer 4 skills metadata cards. Each entry is a
+ * `<skill ... />` self-closing tag so the model sees only the
+ * progressive-disclosure metadata - the full SKILL.md body is
+ * loaded on-demand via `activate_skill(name)` per RB-04.
+ *
+ * @stable
+ */
+export interface SkillMetadataCard {
+  readonly name: string;
+  readonly description: string;
+  readonly location?: string;
+  /** When `true`, the card is excluded from the assembled prompt. */
+  readonly disableModelInvocation?: boolean;
+}
+
+export function composeLayer4Skills(cards: ReadonlyArray<SkillMetadataCard>): string {
+  const visible = cards.filter((c) => c.disableModelInvocation !== true);
+  if (visible.length === 0) return '';
+  const lines = ['<skills_available>'];
+  for (const card of visible) {
+    const description = ` description="${escapeXmlAttr(card.description)}"`;
+    const location =
+      card.location !== undefined ? ` location="${escapeXmlAttr(card.location)}"` : '';
+    lines.push(`  <skill name="${escapeXmlAttr(card.name)}"${description}${location} />`);
+  }
+  lines.push('</skills_available>');
+  return lines.join('\n');
+}
+
+/**
+ * Render the inbound-sanitization preamble fragment. Emitted
+ * AFTER the cache breakpoint so the Layer 1-4 cache prefix is
+ * unaffected. Caller threads the fragment into the system content
+ * post-Layer 5; the fragment is returned as a string so the
+ * caller does not need to know the cache-breakpoint policy.
+ *
+ * @stable
+ */
+export function composeInboundPreamble(pack: ContextLocalePack): string {
+  return pack.inboundSanitizationPreamble.text;
+}
+
+function escapeXmlAttr(value: string): string {
+  return value
+    .replace(/&/g, '&amp;')
+    .replace(/</g, '&lt;')
+    .replace(/>/g, '&gt;')
+    .replace(/"/g, '&quot;');
+}