@haaaiawd/second-nature 0.1.26 → 0.1.29

package/runtime/storage/services/restore-snapshot-store.d.ts

@@ -0,0 +1,52 @@
+/**
+ * RestoreSnapshotStore — T-SMS.C.6
+ *
+ * Core logic: Capture entity snapshots with a whitelist of 6 restorable kinds.
+ * Automatically excludes sensitive kinds (credential, raw_private_message,
+ * raw_prompt, encryption_key, session_token) per DR-017. Retains only the
+ * most recent 3 snapshots by default.
+ *
+ * Dependencies:
+ * - `StateDatabase` from `../db/index.js`
+ * - `RestoreSnapshot`, `RestorableEntityKind`, `SensitiveExcludedKind`
+ *   from `../../shared/types/v7-entities.js`
+ *
+ * Boundary:
+ * - `captureSnapshot` silently drops any requested sensitive kinds.
+ * - `loadLatestSnapshot` returns the most recent capture.
+ * - `listSnapshots` orders by `captured_at` descending.
+ *
+ * Test coverage: tests/unit/storage/restore-snapshot-store.test.ts
+ */
+import type { StateDatabase } from "../db/index.js";
+import type { RestoreSnapshot, RestorableEntityKind } from "../../shared/types/v7-entities.js";
+export interface RestoreSnapshotStore {
+    captureSnapshot(input: {
+        snapshotId: string;
+        entityWhitelist?: RestorableEntityKind[];
+        payload: Record<string, unknown>;
+        capturedAt?: string;
+    }): Promise<RestoreSnapshot>;
+    loadLatestSnapshot(): Promise<RestoreSnapshot | undefined>;
+    listSnapshots(limit?: number): Promise<RestoreSnapshot[]>;
+    /**
+     * Apply a bounded restore from a captured snapshot.
+     * Loads the matching snapshot (by restoreTarget id, or latest fallback),
+     * then attempts to write each whitelisted entity back into its table.
+     * Sensitive kinds are always skipped. Never restores credential fields.
+     */
+    applyBoundedRestore(input: {
+        restoreTarget: string;
+        fromVersion: string;
+        toVersion: string;
+        entityWhitelist?: RestorableEntityKind[];
+    }): Promise<{
+        ok: boolean;
+        completedEntities: string[];
+        failedEntities: string[];
+        warnings: string[];
+    }>;
+}
+export declare function createRestoreSnapshotStore(database: StateDatabase, options?: {
+    retentionCount?: number;
+}): RestoreSnapshotStore;

package/runtime/storage/services/restore-snapshot-store.js

@@ -0,0 +1,193 @@
+/**
+ * RestoreSnapshotStore — T-SMS.C.6
+ *
+ * Core logic: Capture entity snapshots with a whitelist of 6 restorable kinds.
+ * Automatically excludes sensitive kinds (credential, raw_private_message,
+ * raw_prompt, encryption_key, session_token) per DR-017. Retains only the
+ * most recent 3 snapshots by default.
+ *
+ * Dependencies:
+ * - `StateDatabase` from `../db/index.js`
+ * - `RestoreSnapshot`, `RestorableEntityKind`, `SensitiveExcludedKind`
+ *   from `../../shared/types/v7-entities.js`
+ *
+ * Boundary:
+ * - `captureSnapshot` silently drops any requested sensitive kinds.
+ * - `loadLatestSnapshot` returns the most recent capture.
+ * - `listSnapshots` orders by `captured_at` descending.
+ *
+ * Test coverage: tests/unit/storage/restore-snapshot-store.test.ts
+ */
+const ALL_RESTORABLE_KINDS = [
+    "identity_profile",
+    "agent_goal",
+    "tool_experience",
+    "daily_diary",
+    "dream_output",
+    "narrative_timeline",
+];
+const DEFAULT_EXCLUDED_KINDS = [
+    "credential",
+    "raw_private_message",
+    "raw_prompt",
+    "encryption_key",
+    "session_token",
+];
+const DEFAULT_RETENTION_COUNT = 3;
+function safeParseJson(json, fallback) {
+    try {
+        return JSON.parse(json);
+    }
+    catch {
+        return fallback;
+    }
+}
+function parseSnapshotRow(cols, row) {
+    const get = (name) => row[cols.indexOf(name)];
+    return {
+        snapshotId: get("snapshot_id"),
+        entityWhitelist: safeParseJson(get("entity_whitelist_json") ?? "[]", []),
+        excludedSensitiveKinds: safeParseJson(get("excluded_sensitive_kinds_json") ?? "[]", []),
+        capturedAt: get("captured_at"),
+        payload: safeParseJson(get("payload_json") ?? "{}", {}),
+    };
+}
+export function createRestoreSnapshotStore(database, options = {}) {
+    const { sqlite } = database;
+    const retentionCount = options.retentionCount ?? DEFAULT_RETENTION_COUNT;
+    function trimOldSnapshots() {
+        const countResult = sqlite.exec(`SELECT COUNT(*) as cnt FROM restore_snapshot`);
+        if (countResult.length === 0 ||
+            countResult[0].values.length === 0) {
+            return;
+        }
+        const total = countResult[0].values[0][0] ?? 0;
+        if (total <= retentionCount)
+            return;
+        const toDelete = total - retentionCount;
+        sqlite.exec(`DELETE FROM restore_snapshot
+       WHERE snapshot_id IN (
+         SELECT snapshot_id FROM restore_snapshot
+         ORDER BY captured_at ASC
+         LIMIT ${toDelete}
+       )`);
+    }
+    return {
+        async captureSnapshot(input) {
+            const whitelist = input.entityWhitelist && input.entityWhitelist.length > 0
+                ? input.entityWhitelist.filter((k) => ALL_RESTORABLE_KINDS.includes(k))
+                : [...ALL_RESTORABLE_KINDS];
+            const excluded = [...DEFAULT_EXCLUDED_KINDS];
+            const capturedAt = input.capturedAt ?? new Date().toISOString();
+            sqlite.run(`INSERT INTO restore_snapshot
+         (snapshot_id, entity_whitelist_json, excluded_sensitive_kinds_json, captured_at, payload_json)
+         VALUES (?, ?, ?, ?, ?)`, [
+                input.snapshotId,
+                JSON.stringify(whitelist),
+                JSON.stringify(excluded),
+                capturedAt,
+                JSON.stringify(input.payload),
+            ]);
+            trimOldSnapshots();
+            return {
+                snapshotId: input.snapshotId,
+                entityWhitelist: whitelist,
+                excludedSensitiveKinds: [...excluded],
+                capturedAt,
+                payload: input.payload,
+            };
+        },
+        async loadLatestSnapshot() {
+            const result = sqlite.exec(`SELECT * FROM restore_snapshot
+         ORDER BY captured_at DESC
+         LIMIT 1`);
+            if (result.length === 0 || result[0].values.length === 0) {
+                return undefined;
+            }
+            return parseSnapshotRow(result[0].columns, result[0].values[0]);
+        },
+        async listSnapshots(limit = 10) {
+            const result = sqlite.exec(`SELECT * FROM restore_snapshot
+         ORDER BY captured_at DESC
+         LIMIT ${limit}`);
+            if (result.length === 0 || result[0].values.length === 0) {
+                return [];
+            }
+            return result[0].values.map((row) => parseSnapshotRow(result[0].columns, row));
+        },
+        async applyBoundedRestore(input) {
+            const warnings = [];
+            const completedEntities = [];
+            const failedEntities = [];
+            // 1. Find matching snapshot by exact id, then fallback to latest
+            let snapshot;
+            const exactMatch = sqlite.exec(`SELECT * FROM restore_snapshot WHERE snapshot_id = ? LIMIT 1`, [input.restoreTarget]);
+            if (exactMatch.length > 0 &&
+                exactMatch[0].values.length > 0) {
+                snapshot = parseSnapshotRow(exactMatch[0].columns, exactMatch[0].values[0]);
+            }
+            else {
+                const latestResult = sqlite.exec(`SELECT * FROM restore_snapshot ORDER BY captured_at DESC LIMIT 1`);
+                if (latestResult.length > 0 &&
+                    latestResult[0].values.length > 0) {
+                    snapshot = parseSnapshotRow(latestResult[0].columns, latestResult[0].values[0]);
+                }
+            }
+            if (!snapshot) {
+                return {
+                    ok: false,
+                    completedEntities: [],
+                    failedEntities: [input.restoreTarget],
+                    warnings: ["snapshot_not_found"],
+                };
+            }
+            const whitelist = input.entityWhitelist && input.entityWhitelist.length > 0
+                ? input.entityWhitelist.filter((k) => ALL_RESTORABLE_KINDS.includes(k))
+                : [...snapshot.entityWhitelist];
+            for (const kind of whitelist) {
+                // Never restore sensitive kinds (DR-017)
+                if (DEFAULT_EXCLUDED_KINDS.includes(kind)) {
+                    warnings.push(`skipped_sensitive_kind:${kind}`);
+                    continue;
+                }
+                const kindData = snapshot.payload[kind];
+                if (!kindData) {
+                    // Snapshot whitelist includes this kind but payload has no data for it.
+                    // This is not a failure — the entity simply had no state at capture time.
+                    warnings.push(`payload_missing:${kind}`);
+                    continue;
+                }
+                try {
+                    const rows = Array.isArray(kindData) ? kindData : [kindData];
+                    if (rows.length === 0) {
+                        warnings.push(`empty_payload:${kind}`);
+                        continue;
+                    }
+                    for (const row of rows) {
+                        if (!row || typeof row !== "object")
+                            continue;
+                        const keys = Object.keys(row);
+                        if (keys.length === 0)
+                            continue;
+                        const columns = keys.join(", ");
+                        const placeholders = keys.map(() => "?").join(", ");
+                        const values = keys.map((k) => row[k]);
+                        sqlite.run(`INSERT OR REPLACE INTO ${kind} (${columns}) VALUES (${placeholders})`, values);
+                    }
+                    completedEntities.push(kind);
+                }
+                catch (err) {
+                    const msg = err instanceof Error ? err.message : String(err);
+                    warnings.push(`restore_failed:${kind}:${msg}`);
+                    failedEntities.push(kind);
+                }
+            }
+            return {
+                ok: failedEntities.length === 0,
+                completedEntities,
+                failedEntities,
+                warnings,
+            };
+        },
+    };
+}

package/runtime/storage/services/runtime-secret-anchor-store.d.ts

@@ -0,0 +1,26 @@
+/**
+ * RuntimeSecretAnchorStore — T-SMS.C.6
+ *
+ * Core logic: Stores only locationRef/health/rotationPolicyRef for runtime
+ * secrets. Key plaintext is NEVER persisted (ADR-007). WriteValidationGate
+ * rejects any payload that contains key material.
+ *
+ * Dependencies:
+ * - `StateDatabase` from `../db/index.js`
+ * - `RuntimeSecretAnchor`, `SecretAnchorHealth` from `../../shared/types/v7-entities.js`
+ * - `validateWritePayload` from `./write-validation-gate.js`
+ *
+ * Boundary:
+ * - No key/credential/token/encryption_key field is ever written.
+ * - Gate scans for sensitive field keys and rejects on hit.
+ *
+ * Test coverage: tests/unit/storage/runtime-secret-anchor-store.test.ts
+ */
+import type { StateDatabase } from "../db/index.js";
+import type { RuntimeSecretAnchor } from "../../shared/types/v7-entities.js";
+export interface RuntimeSecretAnchorStore {
+    upsertAnchor(anchor: RuntimeSecretAnchor): Promise<void>;
+    loadAnchor(anchorId: string): Promise<RuntimeSecretAnchor | undefined>;
+    listAnchors(): Promise<RuntimeSecretAnchor[]>;
+}
+export declare function createRuntimeSecretAnchorStore(database: StateDatabase): RuntimeSecretAnchorStore;

package/runtime/storage/services/runtime-secret-anchor-store.js

@@ -0,0 +1,82 @@
+/**
+ * RuntimeSecretAnchorStore — T-SMS.C.6
+ *
+ * Core logic: Stores only locationRef/health/rotationPolicyRef for runtime
+ * secrets. Key plaintext is NEVER persisted (ADR-007). WriteValidationGate
+ * rejects any payload that contains key material.
+ *
+ * Dependencies:
+ * - `StateDatabase` from `../db/index.js`
+ * - `RuntimeSecretAnchor`, `SecretAnchorHealth` from `../../shared/types/v7-entities.js`
+ * - `validateWritePayload` from `./write-validation-gate.js`
+ *
+ * Boundary:
+ * - No key/credential/token/encryption_key field is ever written.
+ * - Gate scans for sensitive field keys and rejects on hit.
+ *
+ * Test coverage: tests/unit/storage/runtime-secret-anchor-store.test.ts
+ */
+import { validateWritePayload } from "./write-validation-gate.js";
+export function createRuntimeSecretAnchorStore(database) {
+    const { sqlite } = database;
+    return {
+        async upsertAnchor(anchor) {
+            // Guard: reject any payload that hints at key material
+            const gate = validateWritePayload({
+                anchorId: anchor.anchorId,
+                locationRef: anchor.locationRef,
+                health: anchor.health,
+                rotationPolicyRef: anchor.rotationPolicyRef,
+                updatedAt: anchor.updatedAt,
+            });
+            if (!gate.ok) {
+                throw new Error(`RuntimeSecretAnchor rejected by gate: ${gate.reason}`);
+            }
+            sqlite.run(`INSERT INTO runtime_secret_anchor
+         (anchor_id, location_ref, health, rotation_policy_ref, updated_at)
+         VALUES (?, ?, ?, ?, ?)
+         ON CONFLICT(anchor_id) DO UPDATE SET
+           location_ref = excluded.location_ref,
+           health = excluded.health,
+           rotation_policy_ref = excluded.rotation_policy_ref,
+           updated_at = excluded.updated_at`, [
+                anchor.anchorId,
+                anchor.locationRef,
+                anchor.health,
+                anchor.rotationPolicyRef ?? null,
+                anchor.updatedAt,
+            ]);
+        },
+        async loadAnchor(anchorId) {
+            const result = sqlite.exec(`SELECT * FROM runtime_secret_anchor WHERE anchor_id = ?`, [anchorId]);
+            if (result.length === 0 || result[0].values.length === 0) {
+                return undefined;
+            }
+            const cols = result[0].columns;
+            const get = (row, name) => row[cols.indexOf(name)];
+            const row = result[0].values[0];
+            return {
+                anchorId: get(row, "anchor_id"),
+                locationRef: get(row, "location_ref"),
+                health: get(row, "health"),
+                rotationPolicyRef: get(row, "rotation_policy_ref") ?? undefined,
+                updatedAt: get(row, "updated_at"),
+            };
+        },
+        async listAnchors() {
+            const result = sqlite.exec(`SELECT * FROM runtime_secret_anchor ORDER BY updated_at DESC`);
+            if (result.length === 0 || result[0].values.length === 0) {
+                return [];
+            }
+            const cols = result[0].columns;
+            const get = (row, name) => row[cols.indexOf(name)];
+            return result[0].values.map((row) => ({
+                anchorId: get(row, "anchor_id"),
+                locationRef: get(row, "location_ref"),
+                health: get(row, "health"),
+                rotationPolicyRef: get(row, "rotation_policy_ref") ?? undefined,
+                updatedAt: get(row, "updated_at"),
+            }));
+        },
+    };
+}

package/runtime/storage/services/tool-experience-store.d.ts

@@ -0,0 +1,25 @@
+/**
+ * ToolExperienceStore + CapabilityProbeResultStore — T-SMS.C.5
+ *
+ * Core logic: Append-only ToolExperience rows (outcome/failureClass/latencyMs/
+ * evidenceQuality/sourceRefs/triggerSource). Raw payload rejected by gate.
+ * CapabilityProbeResult stored with capabilityId/actualStatus/httpStatus.
+ * DR-007: failureClass directly from ConnectorResult.
+ * DR-010: triggerSource mandatory.
+ */
+import type { StateDatabase } from "../db/index.js";
+import type { ToolExperience, CapabilityProbeResult } from "../../shared/types/v7-entities.js";
+export interface ToolExperienceStore {
+    appendToolExperience(exp: ToolExperience): Promise<void>;
+    listToolExperience(query: {
+        connectorId?: string;
+        capabilityId?: string;
+        limit?: number;
+    }): Promise<ToolExperience[]>;
+}
+export interface CapabilityProbeResultStore {
+    appendProbeResult(result: CapabilityProbeResult): Promise<void>;
+    listProbeResults(connectorId: string, limit?: number): Promise<CapabilityProbeResult[]>;
+}
+export declare function createToolExperienceStore(database: StateDatabase): ToolExperienceStore;
+export declare function createCapabilityProbeResultStore(database: StateDatabase): CapabilityProbeResultStore;

package/runtime/storage/services/tool-experience-store.js

@@ -0,0 +1,116 @@
+/**
+ * ToolExperienceStore + CapabilityProbeResultStore — T-SMS.C.5
+ *
+ * Core logic: Append-only ToolExperience rows (outcome/failureClass/latencyMs/
+ * evidenceQuality/sourceRefs/triggerSource). Raw payload rejected by gate.
+ * CapabilityProbeResult stored with capabilityId/actualStatus/httpStatus.
+ * DR-007: failureClass directly from ConnectorResult.
+ * DR-010: triggerSource mandatory.
+ */
+import { validateWritePayload } from "./write-validation-gate.js";
+function safeParseJson(json, fallback) {
+    try {
+        return JSON.parse(json);
+    }
+    catch {
+        return fallback;
+    }
+}
+export function createToolExperienceStore(database) {
+    const { sqlite } = database;
+    return {
+        async appendToolExperience(exp) {
+            const gate = validateWritePayload({
+                ...exp,
+                sourceRefs: exp.sourceRefs,
+            }, { runSensitivityScan: false });
+            if (!gate.ok)
+                throw new Error(gate.reason ?? "write_validation_failed");
+            sqlite.run(`INSERT INTO tool_experience
+         (experience_id, connector_id, capability_id, outcome, failure_class,
+          latency_ms, evidence_quality, source_refs_json, trigger_source, created_at)
+         VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`, [
+                exp.experienceId,
+                exp.connectorId,
+                exp.capabilityId,
+                exp.outcome,
+                exp.failureClass ?? null,
+                exp.latencyMs,
+                exp.evidenceQuality,
+                JSON.stringify(exp.sourceRefs),
+                exp.triggerSource,
+                exp.createdAt,
+            ]);
+        },
+        async listToolExperience(query = {}) {
+            let sql = `SELECT * FROM tool_experience WHERE 1=1`;
+            const params = [];
+            if (query.connectorId) {
+                sql += ` AND connector_id = ?`;
+                params.push(query.connectorId);
+            }
+            if (query.capabilityId) {
+                sql += ` AND capability_id = ?`;
+                params.push(query.capabilityId);
+            }
+            sql += ` ORDER BY created_at DESC LIMIT ${query.limit ?? 100}`;
+            const result = sqlite.exec(sql, params);
+            if (result.length === 0 || result[0].values.length === 0)
+                return [];
+            const cols = result[0].columns;
+            const get = (row, name) => row[cols.indexOf(name)];
+            return result[0].values.map((row) => ({
+                experienceId: get(row, "experience_id"),
+                connectorId: get(row, "connector_id"),
+                capabilityId: get(row, "capability_id"),
+                outcome: get(row, "outcome"),
+                failureClass: get(row, "failure_class") ?? undefined,
+                latencyMs: get(row, "latency_ms"),
+                evidenceQuality: get(row, "evidence_quality"),
+                sourceRefs: safeParseJson(get(row, "source_refs_json") ?? "[]", ["store:default"]),
+                triggerSource: get(row, "trigger_source"),
+                createdAt: get(row, "created_at"),
+            }));
+        },
+    };
+}
+export function createCapabilityProbeResultStore(database) {
+    const { sqlite } = database;
+    return {
+        async appendProbeResult(result) {
+            sqlite.run(`INSERT INTO capability_probe_result
+         (probe_result_id, capability_id, connector_id, actual_status,
+          http_status, sample_response_ref, probe_config_ref, created_at)
+         VALUES (?, ?, ?, ?, ?, ?, ?, ?)`, [
+                result.probeResultId,
+                result.capabilityId,
+                result.connectorId,
+                result.actualStatus,
+                result.httpStatus ?? null,
+                result.sampleResponseRef ?? null,
+                result.probeConfigRef,
+                result.createdAt,
+            ]);
+        },
+        async listProbeResults(connectorId, limit = 10) {
+            const result = sqlite.exec(`SELECT * FROM capability_probe_result
+         WHERE connector_id = ?
+         ORDER BY created_at DESC
+         LIMIT ${limit}`, [connectorId]);
+            if (result.length === 0 || result[0].values.length === 0)
+                return [];
+            const cols = result[0].columns;
+            const get = (row, name) => row[cols.indexOf(name)];
+            return result[0].values.map((row) => ({
+                probeResultId: get(row, "probe_result_id"),
+                capabilityId: get(row, "capability_id"),
+                connectorId: get(row, "connector_id"),
+                actualStatus: get(row, "actual_status"),
+                httpStatus: get(row, "http_status") ?? undefined,
+                sampleResponseRef: get(row, "sample_response_ref") ?? undefined,
+                probeConfigRef: get(row, "probe_config_ref"),
+                createdAt: get(row, "created_at"),
+            }));
+        },
+    };
+}

package/runtime/storage/services/write-validation-gate.d.ts

@@ -0,0 +1,46 @@
+/**
+ * WriteValidationGate — T-SMS.C.1
+ *
+ * Core logic: All write paths MUST pass through this gate before persisting.
+ * Rejects payloads containing sensitive fields, missing source refs on fact
+ * claims, or failing schema/sensitivity scans. Returns structured machine-
+ * readable reason codes (DR-022).
+ *
+ * Dependencies:
+ * - `SourceRef` type from `../../shared/types/source-ref.js`
+ * - v7 entity types for shape awareness
+ *
+ * Boundary:
+ * - This gate is stateless; it inspects payloads but never writes.
+ * - Callers must handle rejection before any DB/artifact write.
+ * - Does NOT redact — redaction is the caller's responsibility after gate
+ *   approval.
+ *
+ * Test coverage: tests/unit/storage/write-validation-gate.test.ts
+ */
+export type WriteValidationFailureReason = "write_validation_failed:credential_detected" | "write_validation_failed:token_detected" | "write_validation_failed:raw_private_content_detected" | "write_validation_failed:raw_prompt_detected" | "write_validation_failed:source_refs_missing" | "write_validation_failed:source_refs_empty" | "write_validation_failed:sensitivity_scan_failed" | "write_validation_failed:schema_validation_failed" | "write_validation_failed:encryption_key_detected" | "write_validation_failed:session_token_detected";
+export interface WriteValidationResult {
+    ok: boolean;
+    reason?: WriteValidationFailureReason;
+    details?: string;
+}
+export interface WriteValidationGateOptions {
+    /** If true, fact-claim-like payloads require sourceRefs. Default true. */
+    requireSourceRefs?: boolean;
+    /** If true, run deep sensitivity scan. Default true. */
+    runSensitivityScan?: boolean;
+    /** If true, scan for sensitive field keys. Default true. */
+    scanFieldKeys?: boolean;
+}
+/**
+ * Validate a write payload before persistence.
+ *
+ * @param payload — the object about to be written
+ * @param options — gate behavior tuning
+ * @returns WriteValidationResult: ok=true to proceed, ok=false with reason
+ */
+export declare function validateWritePayload(payload: unknown, options?: WriteValidationGateOptions): WriteValidationResult;
+/**
+ * Convenience: assert that payload passes gate, else throw.
+ */
+export declare function assertWritePayload(payload: unknown, options?: WriteValidationGateOptions): void;