@haaaiawd/second-nature 0.1.38 → 0.1.40

package/runtime/dream/dream-engine.js

@@ -0,0 +1,306 @@
+/**
+ * Dream Engine — orchestrates the hybrid memory consolidation pipeline.
+ *
+ * Pipeline: load inputs → consolidate (rules) → sample → redact →
+ * optional model insights → merge → validate → write output + trace.
+ *
+ * Contract:
+ * - Input store is never modified.
+ * - Output is always candidate until validation passes and lifecycle port accepts it.
+ * - Budget/redaction/timeout failures degrade gracefully with trace.
+ * Test coverage: tests/integration/dream/t7-1-1-dream-pipeline.test.ts
+ */
+import { consolidateMemory } from "./memory-consolidator.js";
+import { sampleDreamInput } from "./sampler.js";
+import { redactDreamInput } from "./redaction-gate.js";
+import { validateDreamOutput } from "./output-validator.js";
+const DEFAULT_TIMEOUT_MS = 30 * 60 * 1000; // 30min
+const DEFAULT_MAX_CANONICAL = 200;
+export async function runDream(input) {
+    const startedAt = new Date().toISOString();
+    const runId = input.runId;
+    const traceId = input.traceId;
+    const triggerKind = input.triggerKind;
+    const options = input.options ?? {};
+    const operatorTimeoutMs = options.operatorTimeoutMs ?? DEFAULT_TIMEOUT_MS;
+    // ─── 1. Load inputs ────────────────────────────────────────────────────────
+    const inputBundle = await input.statePort.loadDreamInputs({
+        timeWindowDays: options.timeWindowDays ?? 30,
+        evidenceLimit: options.evidenceLimit ?? 1000,
+    });
+    if (inputBundle.evidenceRefs.length === 0 &&
+        inputBundle.chronicleEntryIds.length === 0 &&
+        (inputBundle.inputCounts.memoryEntries ?? 0) === 0) {
+        const trace = buildTrace({
+            traceId,
+            runId,
+            startedAt,
+            inputCounts: inputBundle.inputCounts,
+            fallbackReason: "no_inputs",
+        });
+        await input.tracePort?.recordDreamTrace(trace);
+        return {
+            runId,
+            status: "skipped",
+            trace,
+            fallbackReason: "no_inputs",
+        };
+    }
+    // ─── 2. Rules consolidate ──────────────────────────────────────────────────
+    // For the rules stage we use placeholder summaries derived from refs.
+    // Real integration would load actual summaries from state ports.
+    const evidenceSummaries = inputBundle.evidenceRefs.map((ref, i) => ({
+        id: ref,
+        summary: `evidence:${ref}`,
+        sourceRefs: [{ sourceId: ref, kind: "evidence", url: undefined }],
+        createdAt: new Date().toISOString(),
+    }));
+    const chronicleSummaries = inputBundle.chronicleEntryIds.map((id) => ({
+        id,
+        summary: `chronicle:${id}`,
+        sourceRefs: [{ sourceId: id, kind: "chronicle", url: undefined }],
+        createdAt: new Date().toISOString(),
+    }));
+    const toolExperienceSummaries = (inputBundle.toolExperienceSummaries ?? []).map((te) => ({
+        id: `${te.connectorId}:${te.capabilityId}:${te.outcome}`,
+        summary: `tool_experience:${te.connectorId}:${te.capabilityId}:${te.outcome}:count=${te.count}`,
+        sourceRefs: [{ sourceId: `tool_exp:${te.connectorId}:${te.capabilityId}`, kind: "tool_experience", url: undefined }],
+        createdAt: te.lastRecordedAt,
+    }));
+    const consolidation = consolidateMemory({
+        evidenceSummaries,
+        chronicleSummaries,
+        toolExperienceSummaries,
+        existingEntries: [], // In real use, load from activeMemoryStoreId
+    });
+    // ─── 3. Sample ─────────────────────────────────────────────────────────────
+    const sampling = sampleDreamInput({
+        evidenceSummaries: evidenceSummaries.map((e) => ({
+            id: e.id,
+            summary: e.summary,
+            createdAt: e.createdAt,
+        })),
+        chronicleSummaries: chronicleSummaries.map((c) => ({
+            id: c.id,
+            summary: c.summary,
+            createdAt: c.createdAt,
+        })),
+        evidenceLimit: options.evidenceLimit,
+    });
+    // ─── 4. Redaction ──────────────────────────────────────────────────────────
+    const redaction = redactDreamInput({
+        evidenceSummaries: sampling.sampledEvidenceIds.map((id) => evidenceSummaries.find((e) => e.id === id)?.summary ?? id),
+        chronicleSummaries: sampling.sampledChronicleIds.map((id) => chronicleSummaries.find((c) => c.id === id)?.summary ?? id),
+    });
+    if (!redaction.allowed) {
+        const output = buildOutput({
+            runId,
+            inputMemoryStoreId: inputBundle.activeMemoryStoreId,
+            canonicalEntries: consolidation.entries.slice(0, options.maxCanonicalEntries ?? DEFAULT_MAX_CANONICAL),
+            insights: [],
+            validation: {
+                schemaValid: true,
+                sourceGrounded: true,
+                sensitivityClean: false,
+                unsupportedClaims: [],
+                errors: [redaction.blockedReason ?? "redaction_failed"],
+                checkedAt: new Date().toISOString(),
+            },
+        });
+        output.status = "archived";
+        await input.statePort.writeDreamOutput(output);
+        // DR-023: redaction failure is a validation failure → archived lifecycle.
+        await input.statePort.markDreamOutputLifecycle({
+            outputId: output.outputId,
+            newStatus: "archived",
+            validation: output.validation,
+            updatedAt: new Date().toISOString(),
+        });
+        const trace = buildTrace({
+            traceId,
+            runId,
+            startedAt,
+            inputCounts: inputBundle.inputCounts,
+            fallbackReason: redaction.blockedReason ?? "redaction_failed",
+            sensitivityFailure: true,
+        });
+        await input.tracePort?.recordDreamTrace(trace);
+        return {
+            runId,
+            status: "completed",
+            output,
+            trace,
+            fallbackReason: redaction.blockedReason ?? "redaction_failed",
+        };
+    }
+    // ─── 5. Budget gate ────────────────────────────────────────────────────────
+    let modelResult;
+    let mode = "rules_only";
+    let fallbackReason;
+    let llmCostUsd;
+    if ((input.modelAssistPort || input.modelPort) && input.budgetPort) {
+        const budgetCheck = await input.budgetPort.checkBudget(0.5);
+        if (budgetCheck.allowed) {
+            // ─── 6. Model insights ─────────────────────────────────────────────────
+            try {
+                let modelPromise;
+                if (input.modelAssistPort) {
+                    // DR-027: ModelAssistPort requires RedactedEvidenceBundle brand type.
+                    // Evidence already redacted by redactDreamInput above; construct brand
+                    // bundle directly to avoid double-redaction.
+                    const redactedBundle = {
+                        _brand: "redacted",
+                        evidence: redaction.redactedEvidence,
+                        chronicle: redaction.redactedChronicle,
+                        memory: redaction.redactedMemory,
+                    };
+                    modelPromise = input.modelAssistPort.extractInsights(redactedBundle);
+                }
+                else if (input.modelPort) {
+                    // Deprecated path: DreamModelPort accepts plain object (backward compat).
+                    modelPromise = input.modelPort.extractInsights({
+                        sampledEvidence: redaction.redactedEvidence,
+                        chronicleSummary: redaction.redactedChronicle.join("\n"),
+                        redacted: true,
+                    });
+                }
+                if (!fallbackReason) {
+                    const timeoutPromise = new Promise((_, reject) => {
+                        setTimeout(() => reject(new Error("model_timeout")), operatorTimeoutMs);
+                    });
+                    modelResult = await Promise.race([modelPromise, timeoutPromise]);
+                    mode = "hybrid_llm";
+                    llmCostUsd = modelResult.costUsd;
+                }
+            }
+            catch (err) {
+                const msg = err instanceof Error ? err.message : String(err);
+                if (msg.includes("timeout")) {
+                    fallbackReason = "model_timeout";
+                    mode = "model_skipped";
+                }
+                else {
+                    fallbackReason = "model_error";
+                    mode = "model_skipped";
+                }
+            }
+        }
+        else {
+            fallbackReason = "budget_exceeded";
+            mode = "rules_only";
+        }
+    }
+    else {
+        fallbackReason = "model_port_unavailable";
+        mode = "rules_only";
+    }
+    // ─── 7. Merge ──────────────────────────────────────────────────────────────
+    const canonicalEntries = consolidation.entries.slice(0, options.maxCanonicalEntries ?? DEFAULT_MAX_CANONICAL);
+    const insights = modelResult?.insights ?? [];
+    const output = buildOutput({
+        runId,
+        inputMemoryStoreId: inputBundle.activeMemoryStoreId,
+        canonicalEntries,
+        insights,
+        narrativeUpdate: modelResult?.narrativeUpdate,
+        relationshipUpdate: modelResult?.relationshipUpdate,
+        validation: {
+            schemaValid: true,
+            sourceGrounded: true,
+            sensitivityClean: true,
+            unsupportedClaims: modelResult?.unsupportedClaims ?? [],
+            errors: [],
+            checkedAt: new Date().toISOString(),
+        },
+    });
+    // ─── 8. Validate ───────────────────────────────────────────────────────────
+    const toolExperienceIds = toolExperienceSummaries.map((t) => t.sourceRefs[0].sourceId);
+    const validation = validateDreamOutput({
+        output,
+        inputEvidenceIds: inputBundle.evidenceRefs,
+        inputChronicleIds: inputBundle.chronicleEntryIds,
+        inputToolExperienceIds: toolExperienceIds,
+    });
+    // Update output with validation result
+    output.validation = validation.validation;
+    let outputStatus = "candidate";
+    if (!validation.eligible) {
+        outputStatus = "archived";
+        // If model failed but rules produced something, mark partial
+        if (fallbackReason === "model_timeout") {
+            outputStatus = "partial";
+        }
+    }
+    output.status = outputStatus;
+    // ─── 9. Write output + lifecycle transition ──────────────────────────────────
+    await input.statePort.writeDreamOutput(output);
+    // DR-023: validation pass triggers accepted transition.
+    if (validation.eligible) {
+        try {
+            await input.statePort.markDreamOutputLifecycle({
+                outputId: output.outputId,
+                newStatus: "accepted",
+                validation: validation.validation,
+                updatedAt: new Date().toISOString(),
+            });
+            output.status = "accepted";
+        }
+        catch {
+            // Transition failed (e.g., concurrent modification); keep candidate in memory
+            // but DB remains candidate since transition was rolled back.
+            output.status = "candidate";
+        }
+    }
+    const finishedAt = new Date().toISOString();
+    const durationMs = new Date(finishedAt).getTime() - new Date(startedAt).getTime();
+    const trace = buildTrace({
+        traceId,
+        runId,
+        startedAt,
+        finishedAt,
+        durationMs,
+        inputCounts: inputBundle.inputCounts,
+        fallbackReason,
+        llmCostUsd,
+        validationErrors: validation.validation.errors,
+        timeoutMs: fallbackReason === "model_timeout" ? operatorTimeoutMs : undefined,
+        sensitivityFailure: false,
+    });
+    await input.tracePort?.recordDreamTrace(trace);
+    return {
+        runId,
+        status: "completed",
+        output,
+        trace,
+        fallbackReason,
+    };
+}
+// ─── Helpers ──────────────────────────────────────────────────────────────────
+function buildOutput(params) {
+    return {
+        outputId: `dream_output:${crypto.randomUUID()}`,
+        runId: params.runId,
+        status: "candidate",
+        inputMemoryStoreId: params.inputMemoryStoreId,
+        canonicalEntries: params.canonicalEntries,
+        insights: params.insights,
+        narrativeUpdate: params.narrativeUpdate,
+        relationshipUpdate: params.relationshipUpdate,
+        validation: params.validation,
+    };
+}
+function buildTrace(params) {
+    return {
+        traceId: params.traceId,
+        runId: params.runId,
+        startedAt: params.startedAt,
+        finishedAt: params.finishedAt ?? params.startedAt,
+        durationMs: params.durationMs ?? 0,
+        inputCounts: params.inputCounts,
+        fallbackReason: params.fallbackReason,
+        llmCostUsd: params.llmCostUsd,
+        validationErrors: params.validationErrors,
+        timeoutMs: params.timeoutMs,
+        sensitivityFailure: params.sensitivityFailure,
+    };
+}

package/runtime/dream/dream-input-loader.d.ts

@@ -0,0 +1,37 @@
+/**
+ * DreamInputLoader — T-DQS.C.2 (DR-026: Idempotent Claim Loading)
+ *
+ * Core logic: Load unreferenced QuietClaims as Dream inputs.
+ *
+ * Idempotent mechanism (DR-026):
+ * - Queries daily_diary_index + life_evidence_index for candidate source refs.
+ * - Excludes refs already consumed by accepted dream_output_index projections.
+ * - When Dream lock is held, claims are queued; on next Dream run after lock release,
+ *   they are automatically included (no separate "skipped" tracking needed).
+ * - Subsequent Dream runs exclude already-accepted refs, preventing re-processing.
+ *
+ * Lock semantics:
+ * - Lock TTL is 35min (enforced by DreamScheduler, not this module).
+ * - This loader only reads; lock enforcement is upstream.
+ *
+ * ToolExperience summaries:
+ * - Loads recent tool_experience records aggregated by (connector_id, capability_id, outcome).
+ * - Provides frequency count and last recorded time for Dream insight extraction.
+ *
+ * Contract:
+ * - Returns empty evidenceRefs when no unreferenced claims exist.
+ * - Never fabricates inputs; only reads from existing DB state.
+ *
+ * Performance: O(n) where n = life_evidence_index rows (capped by LIMIT).
+ * Memory: O(m) where m = deduplicated ref count (typically < evidenceLimit).
+ *
+ * Test coverage: tests/unit/dream/dream-input-loader.test.ts
+ */
+import type { Database } from "sql.js";
+import type { DreamStatePort } from "./types.js";
+export interface DreamInputLoaderOptions {
+    database: {
+        sqlite: Database;
+    };
+}
+export declare function createDreamInputLoader(options: DreamInputLoaderOptions): Pick<DreamStatePort, "loadDreamInputs">;

package/runtime/dream/dream-input-loader.js

@@ -0,0 +1,155 @@
+/**
+ * DreamInputLoader — T-DQS.C.2 (DR-026: Idempotent Claim Loading)
+ *
+ * Core logic: Load unreferenced QuietClaims as Dream inputs.
+ *
+ * Idempotent mechanism (DR-026):
+ * - Queries daily_diary_index + life_evidence_index for candidate source refs.
+ * - Excludes refs already consumed by accepted dream_output_index projections.
+ * - When Dream lock is held, claims are queued; on next Dream run after lock release,
+ *   they are automatically included (no separate "skipped" tracking needed).
+ * - Subsequent Dream runs exclude already-accepted refs, preventing re-processing.
+ *
+ * Lock semantics:
+ * - Lock TTL is 35min (enforced by DreamScheduler, not this module).
+ * - This loader only reads; lock enforcement is upstream.
+ *
+ * ToolExperience summaries:
+ * - Loads recent tool_experience records aggregated by (connector_id, capability_id, outcome).
+ * - Provides frequency count and last recorded time for Dream insight extraction.
+ *
+ * Contract:
+ * - Returns empty evidenceRefs when no unreferenced claims exist.
+ * - Never fabricates inputs; only reads from existing DB state.
+ *
+ * Performance: O(n) where n = life_evidence_index rows (capped by LIMIT).
+ * Memory: O(m) where m = deduplicated ref count (typically < evidenceLimit).
+ *
+ * Test coverage: tests/unit/dream/dream-input-loader.test.ts
+ */
+function safeParseJson(json, fallback) {
+    try {
+        const parsed = JSON.parse(json);
+        return parsed ?? fallback;
+    }
+    catch {
+        return fallback;
+    }
+}
+/** Extract ref ids from JSON that may be string[] or {id?, sourceId?}[] */
+function extractRefIdsFromJson(json) {
+    const parsed = safeParseJson(json, []);
+    if (!Array.isArray(parsed))
+        return [];
+    const ids = [];
+    for (const item of parsed) {
+        if (typeof item === "string") {
+            ids.push(item);
+        }
+        else if (item && typeof item === "object" && !Array.isArray(item)) {
+            const obj = item;
+            if (typeof obj.id === "string")
+                ids.push(obj.id);
+            if (typeof obj.sourceId === "string")
+                ids.push(obj.sourceId);
+        }
+    }
+    return ids;
+}
+/** Extract consumed ref ids from canonical_entries_json (CanonicalMemoryEntry[]) */
+function extractConsumedRefIdsFromEntriesJson(json) {
+    const entries = safeParseJson(json, []);
+    if (!Array.isArray(entries))
+        return [];
+    const ids = [];
+    for (const entry of entries) {
+        if (entry && typeof entry === "object" && !Array.isArray(entry)) {
+            const sourceRefs = entry.sourceRefs;
+            if (Array.isArray(sourceRefs)) {
+                for (const sr of sourceRefs) {
+                    if (typeof sr === "string") {
+                        ids.push(sr);
+                    }
+                    else if (sr && typeof sr === "object" && !Array.isArray(sr)) {
+                        const obj = sr;
+                        if (typeof obj.sourceId === "string")
+                            ids.push(obj.sourceId);
+                        if (typeof obj.id === "string")
+                            ids.push(obj.id);
+                    }
+                }
+            }
+        }
+    }
+    return ids;
+}
+export function createDreamInputLoader(options) {
+    const { sqlite } = options.database;
+    return {
+        // async aligns with DreamStatePort.loadDreamInputs signature (Promise<DreamInputBundle>).
+        // All current operations are synchronous (sql.js in-memory), but the contract
+        // reserves the right to use async DB drivers in the future.
+        async loadDreamInputs(query = {}) {
+            // Defaults from 05A_TASKS.md T-DQS.C.2 and dream-quiet-system.md §10.2
+            const timeWindowDays = query.timeWindowDays ?? 30;
+            const evidenceLimit = query.evidenceLimit ?? 1000;
+            const since = new Date(Date.now() - timeWindowDays * 24 * 60 * 60 * 1000).toISOString();
+            // ─── 1. Collect candidate ref ids from daily_diary_index ─────────────────
+            const candidateRefs = new Set();
+            const diaryResult = sqlite.exec(`SELECT source_refs_json FROM daily_diary_index WHERE created_at >= ? ORDER BY created_at DESC`, [since]);
+            for (const row of diaryResult[0]?.values ?? []) {
+                for (const id of extractRefIdsFromJson(String(row[0]))) {
+                    candidateRefs.add(id);
+                }
+            }
+            // ─── 2. Collect candidate ref ids from life_evidence_index ───────────────
+            const evidenceResult = sqlite.exec(`SELECT source_refs_json FROM life_evidence_index WHERE timestamp >= ? ORDER BY timestamp DESC LIMIT ?`, [since, evidenceLimit]);
+            for (const row of evidenceResult[0]?.values ?? []) {
+                for (const id of extractRefIdsFromJson(String(row[0]))) {
+                    candidateRefs.add(id);
+                }
+            }
+            // ─── 3. Collect consumed refs from accepted dream outputs ────────────────
+            const consumedRefs = new Set();
+            const acceptedResult = sqlite.exec(`SELECT canonical_entries_json FROM dream_output_index WHERE status = 'accepted'`);
+            for (const row of acceptedResult[0]?.values ?? []) {
+                for (const id of extractConsumedRefIdsFromEntriesJson(String(row[0]))) {
+                    consumedRefs.add(id);
+                }
+            }
+            // ─── 4. Filter: keep only refs not consumed by accepted projections ──────
+            const evidenceRefs = [...candidateRefs].filter((ref) => !consumedRefs.has(ref));
+            // ─── 5. Load ToolExperience summaries (aggregated by connector/capability/outcome) ─
+            const toolExpResult = sqlite.exec(`SELECT connector_id, capability_id, outcome, COUNT(*) as count, MAX(created_at) as last_recorded_at
+         FROM tool_experience
+         WHERE created_at >= ?
+         GROUP BY connector_id, capability_id, outcome
+         ORDER BY last_recorded_at DESC
+         LIMIT ?`, [since, evidenceLimit]);
+            const toolExperienceSummaries = [];
+            for (const row of toolExpResult[0]?.values ?? []) {
+                toolExperienceSummaries.push({
+                    connectorId: String(row[0]),
+                    capabilityId: String(row[1]),
+                    outcome: String(row[2]),
+                    count: Number(row[3]),
+                    lastRecordedAt: String(row[4]),
+                });
+            }
+            return {
+                evidenceRefs,
+                chronicleEntryIds: [],
+                activeMemoryStoreId: undefined,
+                narrativeSnapshotId: undefined,
+                relationshipSnapshotId: undefined,
+                goalSnapshotIds: [],
+                toolExperienceSummaries,
+                inputCounts: {
+                    evidence: evidenceRefs.length,
+                    chronicle: 0, // T-DQS.C.2 scope: evidence only; chronicle loaded separately
+                    memoryEntries: 0, // T-DQS.C.2 scope: evidence only; memory loaded separately
+                },
+            };
+        },
+    };
+}

package/runtime/dream/dream-scheduler.d.ts

@@ -0,0 +1,75 @@
+/**
+ * Dream Scheduler
+ *
+ * Core logic: trigger Dream runs via cron schedule, evidence threshold, or manual
+ * request. Uses DreamRunLock to prevent concurrent runs on the same workspace/input
+ * window. Operator timeout is enforced via options in the engine call.
+ *
+ * - Cron: simplified — checks if a scheduled window is due (last run + interval).
+ * - Evidence threshold: triggers when evidence count exceeds threshold since last run.
+ * - Manual: always allowed if no active lock.
+ * - Lock: in-memory or port-backed; released after run completes or times out.
+ * Test coverage: tests/integration/dream/t7-1-2-dream-scheduler.test.ts
+ */
+import type { DreamEngineInput } from "./types.js";
+import type { DreamTriggerKind, DreamTracePort, DreamStatePort, DreamModelPort, DreamBudgetPort, ModelAssistPort } from "./types.js";
+export interface SchedulerInput {
+    triggerKind: DreamTriggerKind;
+    runId: string;
+    traceId: string;
+    statePort: DreamStatePort;
+    /** @deprecated Use modelAssistPort (DR-027). */
+    modelPort?: DreamModelPort;
+    modelAssistPort?: ModelAssistPort;
+    tracePort?: DreamTracePort;
+    budgetPort?: DreamBudgetPort;
+    options?: DreamEngineInput["options"];
+    lockPort?: DreamRunLockPort;
+    windowKey?: string;
+}
+export interface DreamRunLockPort {
+    acquireLock(input: {
+        runId: string;
+        windowKey: string;
+        ttlMs: number;
+    }): Promise<{
+        acquired: boolean;
+        existingRunId?: string;
+    }>;
+    releaseLock(input: {
+        runId: string;
+        windowKey: string;
+    }): Promise<void>;
+}
+export interface ScheduleResult {
+    runId: string;
+    status: "started" | "skipped" | "queued";
+    reason?: string;
+}
+export declare function memoryLockPort(): DreamRunLockPort;
+export declare function scheduleDream(input: SchedulerInput): Promise<ScheduleResult>;
+export interface CronPolicy {
+    type: "cron";
+    intervalHours: number;
+    lastRunAt?: string;
+}
+export interface EvidenceThresholdPolicy {
+    type: "evidence_threshold";
+    threshold: number;
+    currentEvidenceCount: number;
+    lastRunEvidenceCount: number;
+}
+export interface ManualPolicy {
+    type: "manual";
+}
+export interface QuietCompletionPolicy {
+    type: "quiet_completion";
+    quietCompletedAt: string;
+    windowStartHour: number;
+    windowEndHour: number;
+}
+export type TriggerPolicy = CronPolicy | EvidenceThresholdPolicy | ManualPolicy | QuietCompletionPolicy;
+export declare function shouldTrigger(policy: TriggerPolicy): {
+    shouldRun: boolean;
+    reason?: string;
+};