@haaaiawd/second-nature 0.1.7 → 0.1.9

package/runtime/observability/services/lived-experience-audit.d.ts

@@ -0,0 +1,97 @@
+import { AppendOnlyAuditStore } from "../audit/append-only-audit-store.js";
+import type { SourceRef } from "../../storage/life-evidence/types.js";
+export type RuntimeScope = "rhythm" | "user_task" | "user_reply";
+export type HeartbeatOutcome = "heartbeat_ok" | "intent_selected" | "denied" | "deferred" | "runtime_carrier_only" | "delivery_unavailable";
+export type DeliveryAuditStatus = "not_requested" | "target_available" | "target_none" | "channel_missing" | "host_unsupported" | "ack_dropped" | "sent" | "failed" | "not_sent_fallback";
+export type GroundingStatus = "pass" | "degraded" | "blocked";
+export interface DecisionTracePayload {
+    decisionId: string;
+    traceId: string;
+    heartbeatId?: string;
+    runtimeScope: RuntimeScope;
+    outcome: HeartbeatOutcome;
+    selectedIntentId?: string;
+    candidateId?: string;
+    rhythmWindowKind?: string;
+    hardGuardVerdict?: "allow" | "deny" | "defer" | "silent";
+    outreachVerdict?: "allow" | "deny" | "defer";
+    deliveryAuditId?: string;
+    reasonCodes: string[];
+    sourceRefs: SourceRef[];
+    snapshotRef?: SourceRef;
+    createdAt: string;
+}
+export interface DeliveryAuditPayload {
+    auditId: string;
+    decisionId: string;
+    traceId: string;
+    target?: "none" | "last" | "explicit";
+    channel?: string;
+    recipientRef?: string;
+    status: DeliveryAuditStatus;
+    messageId?: string;
+    hostProofRef?: SourceRef;
+    fallbackRef?: string;
+    ackDropMatched?: boolean;
+    hostVersion?: string;
+    reasonCodes: string[];
+    createdAt: string;
+}
+export interface SourceCoverageAuditPayload {
+    auditId: string;
+    traceId: string;
+    /** When set, explain index links this audit to the decision timeline. */
+    decisionId?: string;
+    subjectType: "quiet_artifact" | "outreach_draft" | "guidance_payload" | "decision_trace" | "host_report";
+    subjectRef: string;
+    usedSourceRefs: SourceRef[];
+    unresolvedRefs: SourceRef[];
+    coverageRatio: number;
+    unsupportedClaims: string[];
+    status: GroundingStatus;
+    reasonCodes: string[];
+    createdAt: string;
+}
+export interface GuidanceGroundingAuditPayload {
+    auditId: string;
+    traceId: string;
+    decisionId?: string;
+    requestId: string;
+    draftId?: string;
+    sceneType: "outreach" | "quiet_reflection" | "social" | "explain" | "user_reply_continuity" | "fallback_candidate";
+    groundingStatus: GroundingStatus;
+    usedSourceRefs: SourceRef[];
+    unsupportedClaims: string[];
+    guardViolations: string[];
+    deliveryWording?: "sendable" | "not_sent_fallback_candidate";
+    createdAt: string;
+}
+export interface ExplainLinkageSummary {
+    decisionId: string;
+    summary: string;
+    warnings: string[];
+    deliveryStatus?: DeliveryAuditStatus;
+    relatedEventIds: string[];
+}
+export declare class LivedExperienceAuditRecorder {
+    private readonly store;
+    private seq;
+    private readonly explainIndex;
+    constructor(store: AppendOnlyAuditStore);
+    private bumpSequence;
+    private touchDecision;
+    recordDecisionTrace(payload: DecisionTracePayload): {
+        eventId: string;
+    };
+    recordDeliveryAudit(payload: DeliveryAuditPayload): {
+        eventId: string;
+    };
+    recordSourceCoverage(payload: SourceCoverageAuditPayload): {
+        eventId: string;
+    };
+    recordGuidanceGrounding(payload: GuidanceGroundingAuditPayload): {
+        eventId: string;
+    };
+    explainLinkageForDecision(decisionId: string): ExplainLinkageSummary;
+}
+export declare function createLivedExperienceAuditRecorder(store?: AppendOnlyAuditStore): LivedExperienceAuditRecorder;

package/runtime/observability/services/lived-experience-audit.js

@@ -0,0 +1,161 @@
+/**
+ * Decision trace, delivery audit, source coverage, guidance grounding + explain index (T5.2.1).
+ *
+ * Core logic: append-only envelopes with hash chain; explain index links decisionId to events and
+ * flags when delivery audit indicates no user-visible contact (target_none / not_sent_fallback).
+ * Test coverage: tests/unit/observability/lived-experience-audit.test.ts
+ */
+import * as crypto from "node:crypto";
+import { AppendOnlyAuditStore } from "../audit/append-only-audit-store.js";
+import { buildAuditEnvelope } from "../audit/audit-envelope.js";
+function validateDecisionTrace(t) {
+    if (!t.decisionId?.trim())
+        throw new Error("decision_trace_requires_decision_id");
+    if (!t.traceId?.trim())
+        throw new Error("decision_trace_requires_trace_id");
+    if (!t.outcome)
+        throw new Error("decision_trace_requires_outcome");
+}
+function validateDeliveryAudit(a) {
+    if (!a.auditId?.trim())
+        throw new Error("delivery_audit_requires_audit_id");
+    if (!a.decisionId?.trim())
+        throw new Error("delivery_audit_requires_decision_id");
+    if (a.status === "sent") {
+        const ok = Boolean(a.messageId?.trim()) || Boolean(a.hostProofRef);
+        if (!ok)
+            throw new Error("delivery_audit_sent_requires_message_id_or_host_proof_ref");
+    }
+}
+export class LivedExperienceAuditRecorder {
+    store;
+    seq = 0;
+    explainIndex = new Map();
+    constructor(store) {
+        this.store = store;
+    }
+    bumpSequence() {
+        this.seq += 1;
+        return this.seq;
+    }
+    touchDecision(decisionId, traceId, eventId) {
+        let e = this.explainIndex.get(decisionId);
+        if (!e) {
+            e = { traceIds: new Set(), eventIds: [], deliveryStatuses: [], fallbackRefs: [], noUserVisibleContact: false };
+            this.explainIndex.set(decisionId, e);
+        }
+        e.traceIds.add(traceId);
+        e.eventIds.push(eventId);
+        return e;
+    }
+    recordDecisionTrace(payload) {
+        validateDecisionTrace(payload);
+        const seq = this.bumpSequence();
+        const envelope = buildAuditEnvelope({
+            family: "heartbeat.decision",
+            plane: "decision",
+            traceId: payload.traceId,
+            sequence: seq,
+            payload,
+            previousHash: this.store.lastRecordHash(),
+            eventId: crypto.randomUUID(),
+            createdAt: payload.createdAt,
+        });
+        this.store.append(envelope);
+        const entry = this.touchDecision(payload.decisionId, payload.traceId, envelope.eventId);
+        if (payload.outcome === "heartbeat_ok" &&
+            payload.reasonCodes.some((c) => c.includes("target_none") || c === "target_none")) {
+            entry.noUserVisibleContact = true;
+        }
+        return { eventId: envelope.eventId };
+    }
+    recordDeliveryAudit(payload) {
+        validateDeliveryAudit(payload);
+        const seq = this.bumpSequence();
+        const envelope = buildAuditEnvelope({
+            family: "delivery",
+            plane: "delivery",
+            traceId: payload.traceId,
+            sequence: seq,
+            payload,
+            previousHash: this.store.lastRecordHash(),
+            eventId: payload.auditId,
+            createdAt: payload.createdAt,
+        });
+        this.store.append(envelope);
+        const entry = this.touchDecision(payload.decisionId, payload.traceId, envelope.eventId);
+        entry.deliveryStatuses.push(payload.status);
+        if (payload.fallbackRef)
+            entry.fallbackRefs.push(payload.fallbackRef);
+        if (payload.status === "target_none" ||
+            payload.status === "not_sent_fallback" ||
+            payload.status === "channel_missing" ||
+            payload.status === "host_unsupported" ||
+            payload.status === "failed") {
+            entry.noUserVisibleContact = true;
+        }
+        return { eventId: envelope.eventId };
+    }
+    recordSourceCoverage(payload) {
+        const seq = this.bumpSequence();
+        const envelope = buildAuditEnvelope({
+            family: "source_coverage",
+            plane: "source_coverage",
+            traceId: payload.traceId,
+            sequence: seq,
+            payload,
+            previousHash: this.store.lastRecordHash(),
+            eventId: payload.auditId,
+            createdAt: payload.createdAt,
+        });
+        this.store.append(envelope);
+        if (payload.decisionId) {
+            this.touchDecision(payload.decisionId, payload.traceId, envelope.eventId);
+        }
+        return { eventId: envelope.eventId };
+    }
+    recordGuidanceGrounding(payload) {
+        const seq = this.bumpSequence();
+        const envelope = buildAuditEnvelope({
+            family: "guidance.grounding",
+            plane: "source_coverage",
+            traceId: payload.traceId,
+            sequence: seq,
+            payload,
+            previousHash: this.store.lastRecordHash(),
+            eventId: payload.auditId,
+            createdAt: payload.createdAt,
+        });
+        this.store.append(envelope);
+        if (payload.decisionId) {
+            this.touchDecision(payload.decisionId, payload.traceId, envelope.eventId);
+        }
+        return { eventId: envelope.eventId };
+    }
+    explainLinkageForDecision(decisionId) {
+        const entry = this.explainIndex.get(decisionId);
+        const warnings = [];
+        if (!entry) {
+            return {
+                decisionId,
+                summary: "no_audit_events_indexed_for_decision",
+                warnings: ["no_indexed_events"],
+                relatedEventIds: [],
+            };
+        }
+        if (entry.noUserVisibleContact) {
+            warnings.push("no_user_visible_contact_claim_prohibited");
+        }
+        const lastDelivery = entry.deliveryStatuses[entry.deliveryStatuses.length - 1];
+        return {
+            decisionId,
+            summary: `indexed_events=${entry.eventIds.length};delivery=${lastDelivery ?? "unknown"}`,
+            warnings,
+            deliveryStatus: lastDelivery,
+            relatedEventIds: [...entry.eventIds],
+        };
+    }
+}
+export function createLivedExperienceAuditRecorder(store) {
+    return new LivedExperienceAuditRecorder(store ?? new AppendOnlyAuditStore());
+}

package/runtime/storage/bootstrap/native-sqlite-probe.d.ts

@@ -0,0 +1,7 @@
+export interface NativeSqliteProbeResult {
+    moduleLoadOk: boolean;
+    /** package version when load succeeds */
+    version?: string;
+    errorMessage?: string;
+}
+export declare function probeNativeSqliteLoad(): NativeSqliteProbeResult;

package/runtime/storage/bootstrap/native-sqlite-probe.js

@@ -0,0 +1,28 @@
+/**
+ * Optional better-sqlite3 load probe (T4.1.4). Second Nature state DB currently uses sql.js only;
+ * this probe records whether the native module can load for packaging / host diagnostics.
+ */
+import { createRequire } from "node:module";
+const require = createRequire(import.meta.url);
+export function probeNativeSqliteLoad() {
+    try {
+        const Database = require("better-sqlite3");
+        const db = new Database(":memory:");
+        db.close();
+        let version;
+        try {
+            const pkg = require("better-sqlite3/package.json");
+            version = pkg.version;
+        }
+        catch {
+            version = undefined;
+        }
+        return { moduleLoadOk: true, version };
+    }
+    catch (error) {
+        return {
+            moduleLoadOk: false,
+            errorMessage: error instanceof Error ? error.message : String(error),
+        };
+    }
+}

package/runtime/storage/bootstrap/repair-gate.d.ts

@@ -0,0 +1,17 @@
+import type { StateDatabase } from "../db/index.js";
+import { createRepairAndBackupService, type RepairAndBackupOptions } from "../services/repair-and-backup.js";
+export interface RepairStateIndexesOptions {
+    startupGate?: boolean;
+    workspaceRoot: string;
+    /** When true, also runs asset registry repair + backup (existing repair service) */
+    reconcileAssets?: boolean;
+    assetRepairOptions?: RepairAndBackupOptions;
+}
+export type RepairGateStatus = "ok" | "repair_required";
+export interface RepairSummary {
+    status: RepairGateStatus;
+    repairedEvidenceIndexRows: number;
+    repairNotes: string[];
+    assetRepair?: Awaited<ReturnType<ReturnType<typeof createRepairAndBackupService>["runStartupRepair"]>>;
+}
+export declare function repairStateIndexes(state: StateDatabase, options: RepairStateIndexesOptions): Promise<RepairSummary>;

package/runtime/storage/bootstrap/repair-gate.js

@@ -0,0 +1,71 @@
+/**
+ * Startup repair gate for state indexes (T4.1.3) — reconciles life evidence filesystem vs SQLite index.
+ *
+ * Core logic: scan `.second-nature/evidence/*.json`; on startupGate, corrupt JSON fails closed with repair_required;
+ * otherwise backfill missing `life_evidence_index` rows from parsed artifacts.
+ *
+ * Test coverage: tests/unit/storage/repair-gate.test.ts
+ */
+import fs from "node:fs";
+import path from "node:path";
+import { eq } from "drizzle-orm";
+import { lifeEvidenceIndex } from "../db/schema/life-evidence-index.js";
+import { createRepairAndBackupService, } from "../services/repair-and-backup.js";
+export async function repairStateIndexes(state, options) {
+    const notes = [];
+    let repaired = 0;
+    const evidenceDir = path.join(options.workspaceRoot, ".second-nature", "evidence");
+    if (!fs.existsSync(evidenceDir)) {
+        notes.push("no_evidence_dir");
+    }
+    else {
+        const entries = fs.readdirSync(evidenceDir);
+        for (const name of entries) {
+            if (!name.endsWith(".json") || name.startsWith(".")) {
+                continue;
+            }
+            const abs = path.join(evidenceDir, name);
+            let parsed;
+            try {
+                parsed = JSON.parse(fs.readFileSync(abs, "utf-8"));
+            }
+            catch {
+                if (options.startupGate) {
+                    return {
+                        status: "repair_required",
+                        repairedEvidenceIndexRows: repaired,
+                        repairNotes: [...notes, `corrupt_evidence_json:${name}`],
+                    };
+                }
+                notes.push(`skip_corrupt:${name}`);
+                continue;
+            }
+            const existing = await state.db.select().from(lifeEvidenceIndex).where(eq(lifeEvidenceIndex.id, parsed.id)).limit(1);
+            if (existing.length === 0) {
+                await state.db.insert(lifeEvidenceIndex).values({
+                    id: parsed.id,
+                    timestamp: parsed.timestamp,
+                    evidenceType: parsed.evidenceType,
+                    sensitivity: parsed.sensitivity,
+                    producer: parsed.producer,
+                    artifactPath: path.join(".second-nature", "evidence", `${parsed.id}.json`).replace(/\\/g, "/"),
+                    platformId: parsed.platformId ?? null,
+                    sourceRefsJson: JSON.stringify(parsed.sourceRefs),
+                });
+                repaired += 1;
+                notes.push(`reindexed:${parsed.id}`);
+            }
+        }
+    }
+    let assetRepair;
+    if (options.reconcileAssets) {
+        const svc = createRepairAndBackupService(state);
+        assetRepair = await svc.runStartupRepair(options.assetRepairOptions ?? {});
+    }
+    return {
+        status: "ok",
+        repairedEvidenceIndexRows: repaired,
+        repairNotes: notes,
+        assetRepair,
+    };
+}

package/runtime/storage/bootstrap/storage-mode-smoke.d.ts

@@ -0,0 +1,38 @@
+import { probeNativeSqliteLoad } from "./native-sqlite-probe.js";
+export interface StorageModeSmokeSemantics {
+    sqlJs: {
+        walAssumed: false;
+        journalConcurrencyNotes: string;
+        backupNotes: string;
+        repairNotes: string;
+    };
+    nativeSqliteWhenAvailable: {
+        journalConcurrencyNotes: string;
+        backupNotes: string;
+    };
+}
+export interface StorageModeSmokeRepairFixtureResult {
+    ran: boolean;
+    workspaceRoot?: string;
+    repairStatus?: "ok" | "repair_required";
+    repairedEvidenceIndexRows?: number;
+    repairNotes?: string[];
+}
+export interface StorageModeSmokeReport {
+    generatedAt: string;
+    /** Implementation backing `createStateDatabase` today — wasm sql.js, not native WAL */
+    runtimeIndexDriver: "sql_js";
+    nativeSqliteProbe: ReturnType<typeof probeNativeSqliteLoad> & {
+        /** Current code path does not use native driver even when load succeeds */
+        runtimeUsesNativeDriver: false;
+    };
+    semantics: StorageModeSmokeSemantics;
+    repairFromArtifactsFixture?: StorageModeSmokeRepairFixtureResult;
+}
+export interface RunStorageModeSmokeOptions {
+    /** Required when runRepairFixture is true — temp dir created if omitted */
+    workspaceRoot?: string;
+    /** Run artifact→index backfill smoke (sql.js path); uses temp workspace when workspaceRoot unset */
+    runRepairFixture?: boolean;
+}
+export declare function runStorageModeSmoke(options?: RunStorageModeSmokeOptions): Promise<StorageModeSmokeReport>;

package/runtime/storage/bootstrap/storage-mode-smoke.js

@@ -0,0 +1,85 @@
+/**
+ * Storage driver mode smoke report (T4.1.4 / state-system §12.1.1).
+ *
+ * Actual runtime: `createStateDatabase` uses sql.js — no WAL; persistence via export + explicit flush on close.
+ * Native SQLite (better-sqlite3) is probed separately; switching drivers is out of scope — report stays honest.
+ */
+import fs from "node:fs";
+import os from "node:os";
+import path from "node:path";
+import { eq } from "drizzle-orm";
+import { appendLifeEvidence } from "../life-evidence/append-life-evidence.js";
+import { createStateDatabase } from "../db/index.js";
+import { lifeEvidenceIndex } from "../db/schema/life-evidence-index.js";
+import { repairStateIndexes } from "./repair-gate.js";
+import { probeNativeSqliteLoad } from "./native-sqlite-probe.js";
+const SEMANTICS = {
+    sqlJs: {
+        walAssumed: false,
+        journalConcurrencyNotes: "sql.js (WASM) has no host SQLite WAL; treat writes as a single-writer/single connection path with explicit export() flush on close (design: single-writer queue + atomic artifact write + explicit flush) — do not assume POSIX WAL or multi-writer journal semantics.",
+        backupNotes: "Backup for sql.js mode: export DB binary via sqlite.export(), pair with workspace artifact manifest; avoid copying a live in-memory DB without explicit flush/export.",
+        repairNotes: "Index rebuild uses canonical filesystem evidence artifacts under .second-nature/evidence — repairStateIndexes can repopulate life_evidence_index without WAL replay.",
+    },
+    nativeSqliteWhenAvailable: {
+        journalConcurrencyNotes: "Native better-sqlite3 may enable WAL where supported + busy timeout — applicable only if a future native-backed StateDatabase is wired.",
+        backupNotes: "Preferred: SQLite Backup API / VACUUM INTO plus artifact manifest — only when native driver owns the DB file.",
+    },
+};
+async function runRepairFixture(workspaceRoot) {
+    const state = createStateDatabase(":memory:");
+    await appendLifeEvidence(state, workspaceRoot, {
+        timestamp: new Date().toISOString(),
+        evidenceType: "platform_browse",
+        summary: "storage-mode-smoke fixture",
+        sourceRefs: [{ id: "smoke-ref", kind: "platform_item", uri: "platform://storage-smoke" }],
+        sensitivity: "public",
+        producer: "state-system",
+    });
+    const rows = await state.db.select().from(lifeEvidenceIndex);
+    const evidenceId = rows[0]?.id;
+    if (!evidenceId) {
+        state.close();
+        return {
+            ran: true,
+            workspaceRoot,
+            repairStatus: "repair_required",
+            repairNotes: ["fixture_failed_no_index_after_append"],
+        };
+    }
+    await state.db.delete(lifeEvidenceIndex).where(eq(lifeEvidenceIndex.id, evidenceId));
+    const repair = await repairStateIndexes(state, { startupGate: true, workspaceRoot });
+    state.close();
+    return {
+        ran: true,
+        workspaceRoot,
+        repairStatus: repair.status,
+        repairedEvidenceIndexRows: repair.repairedEvidenceIndexRows,
+        repairNotes: repair.repairNotes,
+    };
+}
+export async function runStorageModeSmoke(options = {}) {
+    const nativeSqliteProbe = probeNativeSqliteLoad();
+    let repairFromArtifactsFixture;
+    if (options.runRepairFixture) {
+        const ws = options.workspaceRoot ??
+            fs.mkdtempSync(path.join(os.tmpdir(), "sn-storage-smoke-"));
+        try {
+            repairFromArtifactsFixture = await runRepairFixture(ws);
+        }
+        finally {
+            if (!options.workspaceRoot) {
+                fs.rmSync(ws, { recursive: true, force: true });
+            }
+        }
+    }
+    return {
+        generatedAt: new Date().toISOString(),
+        runtimeIndexDriver: "sql_js",
+        nativeSqliteProbe: {
+            ...nativeSqliteProbe,
+            runtimeUsesNativeDriver: false,
+        },
+        semantics: SEMANTICS,
+        repairFromArtifactsFixture,
+    };
+}

package/runtime/storage/db/index.js

@@ -22,8 +22,19 @@ const STATE_SCHEMA_SQL = `
     platform_id TEXT PRIMARY KEY,
     social_daily_limit INTEGER NOT NULL,
     quiet_enabled INTEGER NOT NULL,
+    outreach_daily_budget INTEGER NOT NULL DEFAULT 2,
     updated_at TEXT NOT NULL
   );
+  CREATE TABLE IF NOT EXISTS life_evidence_index (
+    id TEXT PRIMARY KEY,
+    timestamp TEXT NOT NULL,
+    evidence_type TEXT NOT NULL,
+    sensitivity TEXT NOT NULL,
+    producer TEXT NOT NULL,
+    artifact_path TEXT NOT NULL,
+    platform_id TEXT,
+    source_refs_json TEXT NOT NULL
+  );
   CREATE TABLE IF NOT EXISTS asset_registry (
     id TEXT PRIMARY KEY,
     kind TEXT NOT NULL,
@@ -64,6 +75,30 @@ const STATE_SCHEMA_SQL = `
     kind TEXT NOT NULL,
     created_at TEXT NOT NULL
   );
+  CREATE TABLE IF NOT EXISTS delivery_attempts (
+    attempt_id TEXT PRIMARY KEY,
+    decision_id TEXT NOT NULL,
+    target TEXT,
+    channel TEXT,
+    status TEXT NOT NULL,
+    message_id TEXT,
+    host_proof_ref_json TEXT,
+    error_class TEXT,
+    fallback_ref TEXT,
+    created_at TEXT NOT NULL
+  );
+  CREATE INDEX IF NOT EXISTS delivery_attempt_decision_idx ON delivery_attempts(decision_id);
+  CREATE TABLE IF NOT EXISTS operator_fallback_artifacts (
+    fallback_ref TEXT PRIMARY KEY,
+    decision_id TEXT NOT NULL,
+    status TEXT NOT NULL,
+    reason TEXT NOT NULL,
+    source_refs_json TEXT NOT NULL,
+    candidate_message TEXT,
+    next_step TEXT NOT NULL,
+    created_at TEXT NOT NULL
+  );
+  CREATE INDEX IF NOT EXISTS operator_fallback_decision_idx ON operator_fallback_artifacts(decision_id);
 `;
 function resolveDbPath(filename) {
     if (path.isAbsolute(filename) || filename === ":memory:") {
@@ -79,6 +114,20 @@ function resolveDbPath(filename) {
 }
 function bootstrapStateSchema(sqlite) {
     sqlite.exec(STATE_SCHEMA_SQL);
+    applyStateSchemaMigrations(sqlite);
+}
+function applyStateSchemaMigrations(sqlite) {
+    const migrations = [
+        "ALTER TABLE policy_records ADD COLUMN outreach_daily_budget INTEGER NOT NULL DEFAULT 2",
+    ];
+    for (const sql of migrations) {
+        try {
+            sqlite.exec(sql);
+        }
+        catch {
+            /* duplicate column / already migrated */
+        }
+    }
 }
 export function createStateDatabase(filename = "state.db") {
     const dbPath = resolveDbPath(filename);