@haaaiawd/second-nature 0.1.27 → 0.1.29

package/runtime/observability/redaction/policy.js

@@ -10,6 +10,8 @@ export const REDACTION_CONFIG = {
         "bearer_token",
         "authorization",
         "node_secret",
+        "encryption_key", // v7
+        "key_material", // v7
     ],
     eraseFieldNames: [
         "full_message",
@@ -19,12 +21,16 @@ export const REDACTION_CONFIG = {
         "system_prompt",
         "completion",
         "response_content",
+        "raw_payload", // v7
+        "credential_value", // v7
+        "raw_prompt", // v7
     ],
     hashFieldNames: [
         "user_id",
         "session_id",
         "trace_id",
         "content_hash",
+        "message_hash", // v7
     ],
     sensitivityLevels: ["public", "internal", "confidential", "restricted"],
 };
@@ -40,6 +46,8 @@ export const DEFAULT_REDACTION_POLICY = {
         { fieldName: "bearer_token", action: "mask" },
         { fieldName: "authorization", action: "mask" },
         { fieldName: "node_secret", action: "mask" },
+        { fieldName: "encryption_key", action: "mask" },
+        { fieldName: "key_material", action: "mask" },
         { fieldName: "full_message", action: "erase" },
         { fieldName: "full_post", action: "erase" },
         { fieldName: "private_message", action: "erase" },
@@ -47,7 +55,11 @@ export const DEFAULT_REDACTION_POLICY = {
         { fieldName: "system_prompt", action: "erase" },
         { fieldName: "completion", action: "erase" },
         { fieldName: "response_content", action: "erase" },
+        { fieldName: "raw_payload", action: "erase" },
+        { fieldName: "credential_value", action: "erase" },
+        { fieldName: "raw_prompt", action: "erase" },
         { fieldName: "content_hash", action: "hash" },
+        { fieldName: "message_hash", action: "hash" },
     ],
     fieldOverrides: {},
     maxFieldLength: 500,
@@ -69,3 +81,65 @@ export function getFieldRedactionRule(fieldName, policy = DEFAULT_REDACTION_POLI
     }
     return { fieldName, action: "keep" };
 }
+// ─── Unified Redaction Gate (T-OBS.C.1 / DR-033) ────────────────────────────
+import * as crypto from "node:crypto";
+/**
+ * Unified redaction gate — all audit-bound payloads must pass through this
+ * before persistence. Recursively applies mask/erase/hash rules from the
+ * active RedactionPolicy, preserving object shape (erase → null, not delete).
+ *
+ * Boundary:
+ * - Arrays are not recursed (avoid unbounded complexity).
+ * - erase fields become null so downstream JSON schema stays stable.
+ * - hash uses SHA-256 of the stringified original value.
+ */
+export function redactPayload(payload, policy = DEFAULT_REDACTION_POLICY) {
+    const maskedPaths = [];
+    const erasedPaths = [];
+    const hashedPaths = [];
+    function processValue(obj, path) {
+        const result = {};
+        for (const [key, value] of Object.entries(obj)) {
+            const fullPath = path ? `${path}.${key}` : key;
+            const rule = getFieldRedactionRule(key, policy);
+            if (rule.action === "mask") {
+                result[key] = "[MASKED]";
+                maskedPaths.push(fullPath);
+            }
+            else if (rule.action === "erase") {
+                result[key] = null;
+                erasedPaths.push(fullPath);
+            }
+            else if (rule.action === "hash") {
+                result[key] = typeof value === "string"
+                    ? crypto.createHash("sha256").update(value).digest("hex")
+                    : value;
+                hashedPaths.push(fullPath);
+            }
+            else if (value !== null && typeof value === "object" && !Array.isArray(value)) {
+                result[key] = processValue(value, fullPath);
+            }
+            else {
+                result[key] = value;
+            }
+        }
+        return result;
+    }
+    const redactedPayload = processValue(payload, "");
+    return {
+        payload: redactedPayload,
+        manifest: {
+            maskedPaths,
+            erasedPaths,
+            hashedPaths,
+            sensitivity: inferSensitivity(maskedPaths, erasedPaths),
+        },
+    };
+}
+function inferSensitivity(masked, erased) {
+    if (erased.length > 0)
+        return "restricted";
+    if (masked.length > 0)
+        return "confidential";
+    return "internal";
+}

package/runtime/observability/services/heartbeat-digest-assembler.d.ts

@@ -0,0 +1,152 @@
+/**
+ * HeartbeatDigestAssembler — T-OBS.C.3 / T-OBS.C.4
+ *
+ * Core logic: aggregate one day's audit events from AppendOnlyAuditStore
+ * into a dashboard-style HeartbeatDigest (connector counts / goal changes /
+ * quiet-dream status / health summary). No outreach phrasing. No raw payload.
+ * No credential content. If no significant events, isNothingSignificant = true.
+ *
+ * T-OBS.C.4 delivery hook:
+ *   An optional DigestDeliveryAdapter can be injected via deps.deliveryAdapter.
+ *   After digest assembly, generateHeartbeatDigest calls adapter.deliver(digest).
+ *   On success: digest.deliveredAt and digest.deliveryProof are populated.
+ *   On failure: digest.deliveryFallbackReason is set; deliveredAt is NOT set.
+ *   Honesty constraint: not_sent is never reported as sent (ADR-007).
+ *
+ * DR-032 degradation:
+ *   If state-memory port is unavailable, goalSummary + quietDreamSummary
+ *   return degraded = true with reason. Other sections (connector / health) unaffected.
+ *
+ * Boundary:
+ * - Reads AppendOnlyAuditStore.list() (in-memory) for connector.attempt + heartbeat.decision
+ *   + dream.trace + delivery audit events.
+ * - Reads optional StateMemoryDigestPort for goal transitions + quiet/dream scheduling state.
+ * - Does NOT write to state-memory (persistence is runtime-ops' responsibility).
+ * - Does NOT use outreach language (NG2 from PRD: not a "reach out to you" message).
+ * - Does NOT push digest itself; delivery is triggered by runtime-ops (NG5 from L0).
+ *   The adapter here is an injected hook used during assembly, not an autonomous push.
+ *
+ * Test coverage:
+ *   tests/unit/observability/heartbeat-digest-assembler.test.ts (T-OBS.C.3)
+ *   tests/integration/observability/digest-delivery.test.ts (T-OBS.C.4)
+ */
+import type { AppendOnlyAuditStore } from "../audit/append-only-audit-store.js";
+export interface ConnectorDaySummary {
+    platformId: string;
+    capability: string;
+    successCount: number;
+    failureCount: number;
+    circuitOpenCount: number;
+    blockedCount: number;
+}
+export interface GoalDaySummary {
+    newGoals: number;
+    completedGoals: number;
+    expiredGoals: number;
+    replacedGoals: number;
+    activeGoals: number;
+    degraded?: boolean;
+    degradedReason?: string;
+}
+export interface QuietDreamDaySummary {
+    quietRuns: number;
+    quietSucceeded: number;
+    dreamRuns: number;
+    dreamAccepted: number;
+    dreamSkipped: number;
+    dreamSkipReasons: string[];
+    degraded?: boolean;
+    degradedReason?: string;
+}
+export interface HealthDaySummary {
+    circuitBreakerChanges: number;
+    deliverySuccessCount: number;
+    deliveryFailureCount: number;
+    auditChainHealthy: boolean;
+}
+export interface DeliveryProofRef {
+    channelId: string;
+    messageHash: string;
+}
+export interface HeartbeatDigest {
+    date: string;
+    generatedAt: string;
+    isNothingSignificant: boolean;
+    connectorSummary: ConnectorDaySummary[];
+    goalSummary: GoalDaySummary;
+    quietDreamSummary: QuietDreamDaySummary;
+    healthSummary: HealthDaySummary;
+    /** Set when delivery succeeded */
+    deliveredAt?: string;
+    /** Proof of successful delivery (channel + message hash, no raw content) */
+    deliveryProof?: DeliveryProofRef;
+    /** Set when delivery failed; status is always "not_sent" in this case */
+    deliveryFallbackReason?: string;
+}
+/** Result from a delivery attempt */
+export interface DigestDeliveryResult {
+    /**
+     * "sent"     — delivery succeeded; proof is populated.
+     * "not_sent" — delivery failed or was skipped; fallbackReason is populated.
+     */
+    status: "sent" | "not_sent";
+    proof?: DeliveryProofRef;
+    /** Human-readable reason why delivery was not sent */
+    fallbackReason?: string;
+    deliveredAt?: string;
+}
+/**
+ * Adapter injected by runtime-ops to perform channel-specific delivery.
+ * The adapter is responsible for the actual push (Feishu DM / dashboard / etc.).
+ * It must never declare "sent" without a verifiable proof.
+ */
+export interface DigestDeliveryAdapter {
+    deliver(digest: HeartbeatDigest): Promise<DigestDeliveryResult>;
+}
+/** Port for reading goal and quiet/dream scheduling state from state-memory. */
+export interface StateMemoryDigestPort {
+    queryGoalTransitions(date: string): Promise<{
+        newGoals: number;
+        completedGoals: number;
+        expiredGoals: number;
+        replacedGoals: number;
+        activeGoals: number;
+    }>;
+    queryQuietDreamStatus(date: string): Promise<{
+        quietRuns: number;
+        quietSucceeded: number;
+        dreamRuns: number;
+        dreamAccepted: number;
+        dreamSkipped: number;
+        dreamSkipReasons: string[];
+    }>;
+}
+export interface HeartbeatDigestAssemblerDeps {
+    auditStore: AppendOnlyAuditStore;
+    stateMemoryPort?: StateMemoryDigestPort;
+    /**
+     * Optional delivery adapter (T-OBS.C.4).
+     * When provided, the assembled digest is passed to adapter.deliver() after assembly.
+     * Delivery result (proof / fallback) is merged back into the returned digest.
+     * Delivery failure does NOT throw — the assembled digest is still returned,
+     * with deliveryFallbackReason set.
+     */
+    deliveryAdapter?: DigestDeliveryAdapter;
+    /** Override for testability */
+    now?: () => string;
+}
+/**
+ * Generate a HeartbeatDigest for the given date (YYYY-MM-DD).
+ *
+ * Aggregates connector attempts, heartbeat decisions, dream traces, and delivery
+ * audit events from the in-memory audit store. Goal transitions and quiet/dream
+ * scheduling state are loaded from state-memory via the optional port (DR-032
+ * degradation applied if unavailable).
+ *
+ * If deps.deliveryAdapter is provided (T-OBS.C.4), the assembled digest is
+ * passed to the adapter after assembly. Delivery proof or fallback reason is
+ * merged into the returned digest. Delivery failure never causes a throw.
+ *
+ * Does NOT contain outreach language, raw payloads, credentials, or private content.
+ */
+export declare function generateHeartbeatDigest(date: string, deps: HeartbeatDigestAssemblerDeps): Promise<HeartbeatDigest>;

package/runtime/observability/services/heartbeat-digest-assembler.js

@@ -0,0 +1,248 @@
+/**
+ * HeartbeatDigestAssembler — T-OBS.C.3 / T-OBS.C.4
+ *
+ * Core logic: aggregate one day's audit events from AppendOnlyAuditStore
+ * into a dashboard-style HeartbeatDigest (connector counts / goal changes /
+ * quiet-dream status / health summary). No outreach phrasing. No raw payload.
+ * No credential content. If no significant events, isNothingSignificant = true.
+ *
+ * T-OBS.C.4 delivery hook:
+ *   An optional DigestDeliveryAdapter can be injected via deps.deliveryAdapter.
+ *   After digest assembly, generateHeartbeatDigest calls adapter.deliver(digest).
+ *   On success: digest.deliveredAt and digest.deliveryProof are populated.
+ *   On failure: digest.deliveryFallbackReason is set; deliveredAt is NOT set.
+ *   Honesty constraint: not_sent is never reported as sent (ADR-007).
+ *
+ * DR-032 degradation:
+ *   If state-memory port is unavailable, goalSummary + quietDreamSummary
+ *   return degraded = true with reason. Other sections (connector / health) unaffected.
+ *
+ * Boundary:
+ * - Reads AppendOnlyAuditStore.list() (in-memory) for connector.attempt + heartbeat.decision
+ *   + dream.trace + delivery audit events.
+ * - Reads optional StateMemoryDigestPort for goal transitions + quiet/dream scheduling state.
+ * - Does NOT write to state-memory (persistence is runtime-ops' responsibility).
+ * - Does NOT use outreach language (NG2 from PRD: not a "reach out to you" message).
+ * - Does NOT push digest itself; delivery is triggered by runtime-ops (NG5 from L0).
+ *   The adapter here is an injected hook used during assembly, not an autonomous push.
+ *
+ * Test coverage:
+ *   tests/unit/observability/heartbeat-digest-assembler.test.ts (T-OBS.C.3)
+ *   tests/integration/observability/digest-delivery.test.ts (T-OBS.C.4)
+ */
+// ─── Helpers ─────────────────────────────────────────────────────────────────
+function isSameDayUtc(isoTimestamp, dateStr) {
+    // dateStr: "YYYY-MM-DD"
+    return isoTimestamp.startsWith(dateStr);
+}
+function filterByDate(events, family, dateStr) {
+    return events.filter((e) => e.family === family && isSameDayUtc(e.createdAt, dateStr));
+}
+// ─── Aggregation ──────────────────────────────────────────────────────────────
+function aggregateConnectors(events, dateStr) {
+    const connectorEvents = filterByDate(events, "connector.attempt", dateStr);
+    const byKey = new Map();
+    for (const ev of connectorEvents) {
+        const payload = ev.payload;
+        const platformId = payload.platformId ?? "unknown";
+        const capability = payload.capability ?? "unknown";
+        const key = `${platformId}::${capability}`;
+        if (!byKey.has(key)) {
+            byKey.set(key, {
+                platformId,
+                capability,
+                successCount: 0,
+                failureCount: 0,
+                circuitOpenCount: 0,
+                blockedCount: 0,
+            });
+        }
+        const entry = byKey.get(key);
+        switch (payload.outcome) {
+            case "success":
+                entry.successCount++;
+                break;
+            case "failure":
+                entry.failureCount++;
+                break;
+            case "circuit_open":
+                entry.circuitOpenCount++;
+                break;
+            case "blocked":
+                entry.blockedCount++;
+                break;
+            default:
+                // Unknown outcome — still count as an attempt (no-op for summaries)
+                break;
+        }
+    }
+    return Array.from(byKey.values());
+}
+function aggregateHealthSummary(events, dateStr) {
+    const deliveryEvents = filterByDate(events, "delivery", dateStr);
+    const heartbeatEvents = filterByDate(events, "heartbeat.decision", dateStr);
+    const deliverySuccessCount = deliveryEvents.filter((e) => e.payload.outcome === "sent").length;
+    const deliveryFailureCount = deliveryEvents.filter((e) => e.payload.outcome === "failed" ||
+        e.payload.outcome === "not_sent").length;
+    // Count circuit-breaker change events in heartbeat decisions
+    const circuitBreakerChanges = heartbeatEvents.filter((e) => e.payload.outcome === "deferred").length;
+    // Audit chain is healthy if we have consecutive events (simplified: always true here)
+    const auditChainHealthy = true;
+    return {
+        circuitBreakerChanges,
+        deliverySuccessCount,
+        deliveryFailureCount,
+        auditChainHealthy,
+    };
+}
+function aggregateQuietDreamFromAudit(events, dateStr) {
+    const dreamEvents = filterByDate(events, "dream.trace", dateStr);
+    let dreamRuns = 0;
+    let dreamAccepted = 0;
+    let dreamSkipped = 0;
+    const dreamSkipReasons = [];
+    for (const ev of dreamEvents) {
+        const payload = ev.payload;
+        if (payload.event === "dream_started")
+            dreamRuns++;
+        if (payload.event === "dream_accepted")
+            dreamAccepted++;
+        if (payload.event === "dream_skipped") {
+            dreamSkipped++;
+            if (payload.skipReason)
+                dreamSkipReasons.push(payload.skipReason);
+        }
+    }
+    // Quiet stats not yet in audit (narrative.trace not fully wired) — default to 0
+    return {
+        quietRuns: 0,
+        quietSucceeded: 0,
+        dreamRuns,
+        dreamAccepted,
+        dreamSkipped,
+        dreamSkipReasons: [...new Set(dreamSkipReasons)],
+    };
+}
+function isNothingSignificant(connectorSummary, goalSummary, quietDreamSummary, healthSummary) {
+    const hasConnectorActivity = connectorSummary.some((c) => c.successCount + c.failureCount + c.circuitOpenCount + c.blockedCount > 0);
+    const hasGoalActivity = !goalSummary.degraded &&
+        (goalSummary.newGoals +
+            goalSummary.completedGoals +
+            goalSummary.expiredGoals +
+            goalSummary.replacedGoals >
+            0);
+    const hasDreamActivity = !quietDreamSummary.degraded &&
+        (quietDreamSummary.dreamRuns > 0 || quietDreamSummary.quietRuns > 0);
+    const hasDeliveryActivity = healthSummary.deliverySuccessCount + healthSummary.deliveryFailureCount > 0;
+    return (!hasConnectorActivity &&
+        !hasGoalActivity &&
+        !hasDreamActivity &&
+        !hasDeliveryActivity);
+}
+/**
+ * Generate a HeartbeatDigest for the given date (YYYY-MM-DD).
+ *
+ * Aggregates connector attempts, heartbeat decisions, dream traces, and delivery
+ * audit events from the in-memory audit store. Goal transitions and quiet/dream
+ * scheduling state are loaded from state-memory via the optional port (DR-032
+ * degradation applied if unavailable).
+ *
+ * If deps.deliveryAdapter is provided (T-OBS.C.4), the assembled digest is
+ * passed to the adapter after assembly. Delivery proof or fallback reason is
+ * merged into the returned digest. Delivery failure never causes a throw.
+ *
+ * Does NOT contain outreach language, raw payloads, credentials, or private content.
+ */
+export async function generateHeartbeatDigest(date, deps) {
+    const generatedAt = (deps.now ?? (() => new Date().toISOString()))();
+    const { auditStore, stateMemoryPort, deliveryAdapter } = deps;
+    const events = auditStore.list();
+    // Aggregate connector and health from audit
+    const connectorSummary = aggregateConnectors(events, date);
+    const healthSummary = aggregateHealthSummary(events, date);
+    // Goal transitions — via state-memory port (DR-032 degradation)
+    let goalSummary;
+    if (stateMemoryPort) {
+        try {
+            const g = await stateMemoryPort.queryGoalTransitions(date);
+            goalSummary = { ...g };
+        }
+        catch {
+            goalSummary = {
+                newGoals: 0,
+                completedGoals: 0,
+                expiredGoals: 0,
+                replacedGoals: 0,
+                activeGoals: 0,
+                degraded: true,
+                degradedReason: "state_memory_unavailable",
+            };
+        }
+    }
+    else {
+        goalSummary = {
+            newGoals: 0,
+            completedGoals: 0,
+            expiredGoals: 0,
+            replacedGoals: 0,
+            activeGoals: 0,
+        };
+    }
+    // Quiet/Dream status — prefer state-memory port; fallback to audit-based aggregation
+    let quietDreamSummary;
+    if (stateMemoryPort) {
+        try {
+            const qd = await stateMemoryPort.queryQuietDreamStatus(date);
+            quietDreamSummary = { ...qd };
+        }
+        catch {
+            // DR-032: degrade gracefully
+            const fromAudit = aggregateQuietDreamFromAudit(events, date);
+            quietDreamSummary = {
+                ...fromAudit,
+                degraded: true,
+                degradedReason: "state_memory_unavailable",
+            };
+        }
+    }
+    else {
+        quietDreamSummary = aggregateQuietDreamFromAudit(events, date);
+    }
+    const nothingSignificant = isNothingSignificant(connectorSummary, goalSummary, quietDreamSummary, healthSummary);
+    const digest = {
+        date,
+        generatedAt,
+        isNothingSignificant: nothingSignificant,
+        connectorSummary,
+        goalSummary,
+        quietDreamSummary,
+        healthSummary,
+    };
+    // T-OBS.C.4: delivery hook — attempt delivery if adapter is provided
+    if (deliveryAdapter) {
+        try {
+            const result = await deliveryAdapter.deliver(digest);
+            if (result.status === "sent") {
+                // Proof must be present when claiming "sent"
+                if (result.proof) {
+                    digest.deliveredAt = result.deliveredAt ?? generatedAt;
+                    digest.deliveryProof = result.proof;
+                }
+                else {
+                    // Adapter declared "sent" without proof — treat as not_sent (honesty constraint)
+                    digest.deliveryFallbackReason = "delivery_proof_missing";
+                }
+            }
+            else {
+                // status === "not_sent": record fallback reason, never claim sent
+                digest.deliveryFallbackReason = result.fallbackReason ?? "delivery_failed";
+            }
+        }
+        catch (err) {
+            // Delivery threw — absorb error, record fallback, do not rethrow
+            const message = err instanceof Error ? err.message : String(err);
+            digest.deliveryFallbackReason = `delivery_error: ${message}`;
+        }
+    }
+    return digest;
+}

package/runtime/observability/services/lived-experience-audit.js

@@ -57,7 +57,7 @@ export class LivedExperienceAuditRecorder {
             traceId: payload.traceId,
             sequence: seq,
             payload,
-            previousHash: this.store.lastRecordHash(),
+            previousHash: this.store.lastRecordHash("heartbeat.decision"),
             eventId: crypto.randomUUID(),
             createdAt: payload.createdAt,
         });
@@ -78,7 +78,7 @@ export class LivedExperienceAuditRecorder {
             traceId: payload.traceId,
             sequence: seq,
             payload,
-            previousHash: this.store.lastRecordHash(),
+            previousHash: this.store.lastRecordHash("delivery"),
             eventId: payload.auditId,
             createdAt: payload.createdAt,
         });
@@ -105,7 +105,7 @@ export class LivedExperienceAuditRecorder {
             traceId: payload.traceId,
             sequence: seq,
             payload,
-            previousHash: this.store.lastRecordHash(),
+            previousHash: this.store.lastRecordHash("source_coverage"),
             eventId: payload.auditId,
             createdAt: payload.createdAt,
         });
@@ -123,7 +123,7 @@ export class LivedExperienceAuditRecorder {
             traceId: payload.traceId,
             sequence: seq,
             payload,
-            previousHash: this.store.lastRecordHash(),
+            previousHash: this.store.lastRecordHash("guidance.grounding"),
             eventId: payload.auditId,
             createdAt: payload.createdAt,
         });
@@ -141,7 +141,7 @@ export class LivedExperienceAuditRecorder {
             traceId: payload.traceId,
             sequence: seq,
             payload,
-            previousHash: this.store.lastRecordHash(),
+            previousHash: this.store.lastRecordHash("narrative.trace"),
             eventId: crypto.randomUUID(),
             createdAt: payload.createdAt,
         });
@@ -156,7 +156,7 @@ export class LivedExperienceAuditRecorder {
             traceId: payload.traceId,
             sequence: seq,
             payload,
-            previousHash: this.store.lastRecordHash(),
+            previousHash: this.store.lastRecordHash("dream.trace"),
             eventId: crypto.randomUUID(),
             createdAt: payload.finishedAt,
         });