@haaaiawd/second-nature 0.2.6 → 0.2.7
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/openclaw.plugin.json +1 -1
- package/package.json +1 -1
- package/runtime/cli/commands/index.js +9 -0
- package/runtime/connectors/base/normalized-evidence-content.d.ts +71 -0
- package/runtime/connectors/base/normalized-evidence-content.js +273 -0
- package/runtime/connectors/evidence-normalizer.d.ts +4 -3
- package/runtime/connectors/evidence-normalizer.js +128 -23
- package/runtime/core/second-nature/heartbeat/heartbeat-loop.js +15 -0
- package/runtime/core/second-nature/perception/perception-builder.d.ts +3 -1
- package/runtime/core/second-nature/perception/perception-builder.js +98 -44
- package/runtime/core/second-nature/quiet-dream/daily-rhythm-scheduler.d.ts +6 -3
- package/runtime/core/second-nature/quiet-dream/daily-rhythm-scheduler.js +198 -22
- package/runtime/core/second-nature/quiet-dream/quiet-daily-review-builder.d.ts +19 -3
- package/runtime/core/second-nature/quiet-dream/quiet-daily-review-builder.js +189 -12
- package/runtime/shared/types/v8-contracts.d.ts +1 -1
- package/runtime/storage/db/index.js +2 -0
- package/runtime/storage/db/migrations/v8-003-quiet-closure-refs.js +2 -1
- package/runtime/storage/services/write-validation-gate.d.ts +2 -0
- package/runtime/storage/services/write-validation-gate.js +69 -17
- package/runtime/storage/v8-state-stores.d.ts +25 -0
- package/runtime/storage/v8-state-stores.js +87 -1
package/openclaw.plugin.json
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
{
|
|
2
2
|
"id": "second-nature",
|
|
3
3
|
"name": "Second Nature",
|
|
4
|
-
"version": "0.2.
|
|
4
|
+
"version": "0.2.7",
|
|
5
5
|
"description": "OpenClaw native plugin with synchronous surface registration and bundled runtime spine. Set SECOND_NATURE_WORKSPACE_ROOT or tool workspaceRoot to the same path as the agent workspace. Agent inner guide is packaged as agent-inner-guide.md. v7 ops surface: self_health, tool_affordance, heartbeat_digest, snapshot:capture, narrative:diff, timeline, restore, runtime_secret_bootstrap, connector:run, guidance_payload.",
|
|
6
6
|
"activation": {
|
|
7
7
|
"onStartup": true,
|
package/package.json
CHANGED
|
@@ -449,6 +449,15 @@ export function createCliCommands(deps) {
|
|
|
449
449
|
return surface;
|
|
450
450
|
},
|
|
451
451
|
},
|
|
452
|
+
{
|
|
453
|
+
name: "connector_status",
|
|
454
|
+
description: "T1.2.5 — show connector inventory status, trust, and health conflicts",
|
|
455
|
+
execute: async (input) => {
|
|
456
|
+
const surface = await Promise.resolve(opsRouter.dispatch("connector_status", input));
|
|
457
|
+
flush();
|
|
458
|
+
return surface;
|
|
459
|
+
},
|
|
460
|
+
},
|
|
452
461
|
opsCommand("connector:run", "T-ROS.C.3 — manually execute a connector capability outside heartbeat cadence"),
|
|
453
462
|
opsCommand("loop_status", "T-ROS.C.1 — show v8 causal loop health: stalled stage, next action, and stage summaries"),
|
|
454
463
|
opsCommand("self_health", "T-ROS.C.1 — show v7 self-health snapshot and degraded dimensions"),
|
|
@@ -0,0 +1,71 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* NormalizedEvidenceContent — Cross-platform content-bearing evidence envelope.
|
|
3
|
+
*
|
|
4
|
+
* Core logic: Map arbitrary connector read payloads into a stable, source-backed
|
|
5
|
+
* summary structure that perception, Quiet, and Dream can consume. This is a
|
|
6
|
+
* schema boundary, not a judgment layer.
|
|
7
|
+
*
|
|
8
|
+
* Design authority:
|
|
9
|
+
* - `.anws/v8/04_SYSTEM_DESIGN/connector-system.md`
|
|
10
|
+
* - `.anws/v8/04_SYSTEM_DESIGN/perception-judgment-system.md`
|
|
11
|
+
*
|
|
12
|
+
* Dependencies:
|
|
13
|
+
* - none (pure extraction)
|
|
14
|
+
*
|
|
15
|
+
* Boundary:
|
|
16
|
+
* - Does not classify sensitivity.
|
|
17
|
+
* - Does not redact; callers run redaction separately.
|
|
18
|
+
* - Does not persist; callers write EvidenceItem.
|
|
19
|
+
* - Preserves raw values in canonicalText/excerpt so downstream can decide what to keep.
|
|
20
|
+
*
|
|
21
|
+
* Test coverage: tests/unit/connectors/normalized-evidence-content.test.ts
|
|
22
|
+
*/
|
|
23
|
+
export declare const NORMALIZED_EVIDENCE_SCHEMA_VERSION = 1;
|
|
24
|
+
export type EvidenceSourceKind = "post" | "comment" | "profile" | "task" | "event" | "game_state" | "notification" | "document" | "unknown";
|
|
25
|
+
export type SummaryProducer = "connector_rules" | "model_assist" | "operator_supplied";
|
|
26
|
+
export interface EvidenceActor {
|
|
27
|
+
id?: string;
|
|
28
|
+
displayName?: string;
|
|
29
|
+
role?: string;
|
|
30
|
+
}
|
|
31
|
+
export interface NormalizedEvidenceContent {
|
|
32
|
+
schemaVersion: typeof NORMALIZED_EVIDENCE_SCHEMA_VERSION;
|
|
33
|
+
sourceKind: EvidenceSourceKind;
|
|
34
|
+
platformId: string;
|
|
35
|
+
capabilityId: string;
|
|
36
|
+
externalId?: string;
|
|
37
|
+
title?: string;
|
|
38
|
+
summary: string;
|
|
39
|
+
excerpt?: string;
|
|
40
|
+
canonicalText?: string;
|
|
41
|
+
actor?: EvidenceActor;
|
|
42
|
+
url?: string;
|
|
43
|
+
occurredAt?: string;
|
|
44
|
+
observedAt: string;
|
|
45
|
+
tags?: string[];
|
|
46
|
+
entities?: string[];
|
|
47
|
+
metrics?: Record<string, number | string | boolean>;
|
|
48
|
+
rawContentRef?: string;
|
|
49
|
+
summaryProducer: SummaryProducer;
|
|
50
|
+
}
|
|
51
|
+
export interface ExtractEvidenceOptions {
|
|
52
|
+
platformId: string;
|
|
53
|
+
capabilityId: string;
|
|
54
|
+
observedAt?: string;
|
|
55
|
+
summaryProducer?: SummaryProducer;
|
|
56
|
+
/** Max characters for excerpt. Default 240. */
|
|
57
|
+
excerptMaxChars?: number;
|
|
58
|
+
/** Max characters for canonicalText. Default 2000. */
|
|
59
|
+
canonicalTextMaxChars?: number;
|
|
60
|
+
}
|
|
61
|
+
/**
|
|
62
|
+
* Extract a list of content-bearing evidence items from a connector success payload.
|
|
63
|
+
* Returns empty array when data is not an object/array or contains no extractable items.
|
|
64
|
+
*/
|
|
65
|
+
export declare function extractNormalizedEvidenceItems(data: unknown, options: ExtractEvidenceOptions): NormalizedEvidenceContent[];
|
|
66
|
+
/**
|
|
67
|
+
* Compute a stable content hash for deduplication across connector runs.
|
|
68
|
+
* Prefer externalId-based identity; this hash is the fallback.
|
|
69
|
+
*/
|
|
70
|
+
export declare function computeEvidenceContentHash(content: NormalizedEvidenceContent): string;
|
|
71
|
+
export declare function computeEvidenceContentHashSync(content: NormalizedEvidenceContent): string;
|
|
@@ -0,0 +1,273 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* NormalizedEvidenceContent — Cross-platform content-bearing evidence envelope.
|
|
3
|
+
*
|
|
4
|
+
* Core logic: Map arbitrary connector read payloads into a stable, source-backed
|
|
5
|
+
* summary structure that perception, Quiet, and Dream can consume. This is a
|
|
6
|
+
* schema boundary, not a judgment layer.
|
|
7
|
+
*
|
|
8
|
+
* Design authority:
|
|
9
|
+
* - `.anws/v8/04_SYSTEM_DESIGN/connector-system.md`
|
|
10
|
+
* - `.anws/v8/04_SYSTEM_DESIGN/perception-judgment-system.md`
|
|
11
|
+
*
|
|
12
|
+
* Dependencies:
|
|
13
|
+
* - none (pure extraction)
|
|
14
|
+
*
|
|
15
|
+
* Boundary:
|
|
16
|
+
* - Does not classify sensitivity.
|
|
17
|
+
* - Does not redact; callers run redaction separately.
|
|
18
|
+
* - Does not persist; callers write EvidenceItem.
|
|
19
|
+
* - Preserves raw values in canonicalText/excerpt so downstream can decide what to keep.
|
|
20
|
+
*
|
|
21
|
+
* Test coverage: tests/unit/connectors/normalized-evidence-content.test.ts
|
|
22
|
+
*/
|
|
23
|
+
export const NORMALIZED_EVIDENCE_SCHEMA_VERSION = 1;
|
|
24
|
+
// ───────────────────────────────────────────────────────────────
|
|
25
|
+
// String helpers
|
|
26
|
+
// ───────────────────────────────────────────────────────────────
|
|
27
|
+
function isNonEmptyString(value) {
|
|
28
|
+
return typeof value === "string" && value.trim().length > 0;
|
|
29
|
+
}
|
|
30
|
+
function firstNonEmptyString(values) {
|
|
31
|
+
for (const v of values) {
|
|
32
|
+
if (isNonEmptyString(v))
|
|
33
|
+
return v.trim();
|
|
34
|
+
}
|
|
35
|
+
return undefined;
|
|
36
|
+
}
|
|
37
|
+
function truncate(text, maxChars) {
|
|
38
|
+
if (text.length <= maxChars)
|
|
39
|
+
return text;
|
|
40
|
+
return `${text.slice(0, maxChars)}…`;
|
|
41
|
+
}
|
|
42
|
+
function toStringArray(value) {
|
|
43
|
+
if (!Array.isArray(value))
|
|
44
|
+
return [];
|
|
45
|
+
return value
|
|
46
|
+
.map((item) => (typeof item === "string" ? item.trim() : String(item ?? "").trim()))
|
|
47
|
+
.filter((item) => item.length > 0);
|
|
48
|
+
}
|
|
49
|
+
function normalizeDate(value) {
|
|
50
|
+
if (typeof value !== "string" || value.trim().length === 0)
|
|
51
|
+
return undefined;
|
|
52
|
+
const parsed = Date.parse(value);
|
|
53
|
+
if (Number.isNaN(parsed))
|
|
54
|
+
return undefined;
|
|
55
|
+
return new Date(parsed).toISOString();
|
|
56
|
+
}
|
|
57
|
+
// ───────────────────────────────────────────────────────────────
|
|
58
|
+
// Object traversal helpers
|
|
59
|
+
// ───────────────────────────────────────────────────────────────
|
|
60
|
+
function isRecord(value) {
|
|
61
|
+
return value !== null && typeof value === "object" && !Array.isArray(value);
|
|
62
|
+
}
|
|
63
|
+
function findFirstValue(obj, keys) {
|
|
64
|
+
if (!isRecord(obj))
|
|
65
|
+
return undefined;
|
|
66
|
+
for (const key of keys) {
|
|
67
|
+
if (key in obj)
|
|
68
|
+
return obj[key];
|
|
69
|
+
}
|
|
70
|
+
return undefined;
|
|
71
|
+
}
|
|
72
|
+
function findDeepValue(obj, keys) {
|
|
73
|
+
const queue = [obj];
|
|
74
|
+
while (queue.length > 0) {
|
|
75
|
+
const current = queue.shift();
|
|
76
|
+
if (isRecord(current)) {
|
|
77
|
+
const found = findFirstValue(current, keys);
|
|
78
|
+
if (found !== undefined)
|
|
79
|
+
return found;
|
|
80
|
+
for (const v of Object.values(current)) {
|
|
81
|
+
queue.push(v);
|
|
82
|
+
}
|
|
83
|
+
}
|
|
84
|
+
else if (Array.isArray(current)) {
|
|
85
|
+
for (const v of current) {
|
|
86
|
+
queue.push(v);
|
|
87
|
+
}
|
|
88
|
+
}
|
|
89
|
+
}
|
|
90
|
+
return undefined;
|
|
91
|
+
}
|
|
92
|
+
// ───────────────────────────────────────────────────────────────
|
|
93
|
+
// Source kind inference
|
|
94
|
+
// ───────────────────────────────────────────────────────────────
|
|
95
|
+
function inferSourceKind(item) {
|
|
96
|
+
const typeHint = String(findFirstValue(item, ["type", "kind", "sourceKind", "object", "category"]) ?? "").toLowerCase();
|
|
97
|
+
if (typeHint.includes("post"))
|
|
98
|
+
return "post";
|
|
99
|
+
if (typeHint.includes("comment") || typeHint.includes("reply"))
|
|
100
|
+
return "comment";
|
|
101
|
+
if (typeHint.includes("profile") || typeHint.includes("user") || typeHint.includes("author"))
|
|
102
|
+
return "profile";
|
|
103
|
+
if (typeHint.includes("task") || typeHint.includes("issue") || typeHint.includes("todo"))
|
|
104
|
+
return "task";
|
|
105
|
+
if (typeHint.includes("event") || typeHint.includes("activity"))
|
|
106
|
+
return "event";
|
|
107
|
+
if (typeHint.includes("game") || typeHint.includes("match") || typeHint.includes("score"))
|
|
108
|
+
return "game_state";
|
|
109
|
+
if (typeHint.includes("notification") || typeHint.includes("alert"))
|
|
110
|
+
return "notification";
|
|
111
|
+
if (typeHint.includes("document") || typeHint.includes("article") || typeHint.includes("page"))
|
|
112
|
+
return "document";
|
|
113
|
+
return "unknown";
|
|
114
|
+
}
|
|
115
|
+
// ───────────────────────────────────────────────────────────────
|
|
116
|
+
// Field extraction
|
|
117
|
+
// ───────────────────────────────────────────────────────────────
|
|
118
|
+
function extractId(item) {
|
|
119
|
+
const id = findFirstValue(item, ["id", "externalId", "postId", "taskId", "eventId"]);
|
|
120
|
+
if (id === undefined || id === null)
|
|
121
|
+
return undefined;
|
|
122
|
+
return String(id).trim();
|
|
123
|
+
}
|
|
124
|
+
function extractTitle(item) {
|
|
125
|
+
return firstNonEmptyString([
|
|
126
|
+
findFirstValue(item, ["title", "subject", "heading", "name", "displayName"]),
|
|
127
|
+
]);
|
|
128
|
+
}
|
|
129
|
+
function extractBodyText(item) {
|
|
130
|
+
return firstNonEmptyString([
|
|
131
|
+
findFirstValue(item, ["content", "body", "text", "description", "message", "snippet", "summary"]),
|
|
132
|
+
]);
|
|
133
|
+
}
|
|
134
|
+
function extractUrl(item) {
|
|
135
|
+
const url = findFirstValue(item, ["url", "link", "permalink", "href", "webUrl"]);
|
|
136
|
+
if (!isNonEmptyString(url))
|
|
137
|
+
return undefined;
|
|
138
|
+
try {
|
|
139
|
+
new URL(url);
|
|
140
|
+
return url;
|
|
141
|
+
}
|
|
142
|
+
catch {
|
|
143
|
+
return undefined;
|
|
144
|
+
}
|
|
145
|
+
}
|
|
146
|
+
function extractActor(item) {
|
|
147
|
+
const author = findFirstValue(item, ["author", "creator", "user", "actor", "from"]);
|
|
148
|
+
if (!isRecord(author) && !isNonEmptyString(author))
|
|
149
|
+
return undefined;
|
|
150
|
+
if (isNonEmptyString(author)) {
|
|
151
|
+
return { displayName: author };
|
|
152
|
+
}
|
|
153
|
+
const displayName = firstNonEmptyString([
|
|
154
|
+
findFirstValue(author, ["displayName", "name", "username", "handle", "login"]),
|
|
155
|
+
]);
|
|
156
|
+
const id = firstNonEmptyString([findFirstValue(author, ["id", "userId", "accountId"])]);
|
|
157
|
+
const role = firstNonEmptyString([findFirstValue(author, ["role", "type"])]);
|
|
158
|
+
if (!displayName && !id)
|
|
159
|
+
return undefined;
|
|
160
|
+
return { displayName, id, role };
|
|
161
|
+
}
|
|
162
|
+
function extractOccurredAt(item) {
|
|
163
|
+
return normalizeDate(findFirstValue(item, ["createdAt", "publishedAt", "occurredAt", "timestamp", "date", "time"]));
|
|
164
|
+
}
|
|
165
|
+
function extractTags(item) {
|
|
166
|
+
const tags = findFirstValue(item, ["tags", "labels", "categories", "topics"]);
|
|
167
|
+
return toStringArray(tags);
|
|
168
|
+
}
|
|
169
|
+
function extractEntities(item) {
|
|
170
|
+
const entities = findFirstValue(item, ["entities", "mentions", "hashtags", "keywords"]);
|
|
171
|
+
return toStringArray(entities);
|
|
172
|
+
}
|
|
173
|
+
function extractMetrics(item) {
|
|
174
|
+
const metrics = findFirstValue(item, ["metrics", "stats", "counts", "engagement"]);
|
|
175
|
+
if (!isRecord(metrics))
|
|
176
|
+
return undefined;
|
|
177
|
+
const out = {};
|
|
178
|
+
for (const [k, v] of Object.entries(metrics)) {
|
|
179
|
+
if (typeof v === "number" || typeof v === "string" || typeof v === "boolean") {
|
|
180
|
+
out[k] = v;
|
|
181
|
+
}
|
|
182
|
+
}
|
|
183
|
+
return Object.keys(out).length > 0 ? out : undefined;
|
|
184
|
+
}
|
|
185
|
+
// ───────────────────────────────────────────────────────────────
|
|
186
|
+
// Item extraction from connector payload
|
|
187
|
+
// ───────────────────────────────────────────────────────────────
|
|
188
|
+
function extractItems(data) {
|
|
189
|
+
if (Array.isArray(data))
|
|
190
|
+
return data;
|
|
191
|
+
if (!isRecord(data))
|
|
192
|
+
return [];
|
|
193
|
+
for (const key of ["items", "data", "results", "posts", "nodes", "agents", "edges", "entries", "feed"]) {
|
|
194
|
+
const candidate = data[key];
|
|
195
|
+
if (Array.isArray(candidate))
|
|
196
|
+
return candidate;
|
|
197
|
+
}
|
|
198
|
+
// If the payload itself looks like a single item, treat it as one-item array.
|
|
199
|
+
if ("id" in data || "title" in data || "content" in data)
|
|
200
|
+
return [data];
|
|
201
|
+
return [];
|
|
202
|
+
}
|
|
203
|
+
function normalizeSingleItem(item, options) {
|
|
204
|
+
if (!isRecord(item))
|
|
205
|
+
return null;
|
|
206
|
+
const sourceKind = inferSourceKind(item);
|
|
207
|
+
const externalId = extractId(item);
|
|
208
|
+
const title = extractTitle(item);
|
|
209
|
+
const bodyText = extractBodyText(item);
|
|
210
|
+
const url = extractUrl(item);
|
|
211
|
+
const actor = extractActor(item);
|
|
212
|
+
const occurredAt = extractOccurredAt(item);
|
|
213
|
+
const tags = extractTags(item);
|
|
214
|
+
const entities = extractEntities(item);
|
|
215
|
+
const metrics = extractMetrics(item);
|
|
216
|
+
const rawText = bodyText ?? title ?? "[no readable content]";
|
|
217
|
+
const summary = truncate(rawText, 160);
|
|
218
|
+
const excerpt = rawText.length > summary.length ? truncate(rawText, options.excerptMaxChars ?? 240) : undefined;
|
|
219
|
+
const canonicalText = truncate(rawText, options.canonicalTextMaxChars ?? 2000);
|
|
220
|
+
return {
|
|
221
|
+
schemaVersion: NORMALIZED_EVIDENCE_SCHEMA_VERSION,
|
|
222
|
+
sourceKind,
|
|
223
|
+
platformId: options.platformId,
|
|
224
|
+
capabilityId: options.capabilityId,
|
|
225
|
+
externalId,
|
|
226
|
+
title,
|
|
227
|
+
summary,
|
|
228
|
+
excerpt,
|
|
229
|
+
canonicalText,
|
|
230
|
+
actor,
|
|
231
|
+
url,
|
|
232
|
+
occurredAt,
|
|
233
|
+
observedAt: options.observedAt ?? new Date().toISOString(),
|
|
234
|
+
tags,
|
|
235
|
+
entities,
|
|
236
|
+
metrics,
|
|
237
|
+
summaryProducer: options.summaryProducer ?? "connector_rules",
|
|
238
|
+
};
|
|
239
|
+
}
|
|
240
|
+
/**
|
|
241
|
+
* Extract a list of content-bearing evidence items from a connector success payload.
|
|
242
|
+
* Returns empty array when data is not an object/array or contains no extractable items.
|
|
243
|
+
*/
|
|
244
|
+
export function extractNormalizedEvidenceItems(data, options) {
|
|
245
|
+
const items = extractItems(data);
|
|
246
|
+
const out = [];
|
|
247
|
+
for (const item of items) {
|
|
248
|
+
const normalized = normalizeSingleItem(item, options);
|
|
249
|
+
if (normalized)
|
|
250
|
+
out.push(normalized);
|
|
251
|
+
}
|
|
252
|
+
return out;
|
|
253
|
+
}
|
|
254
|
+
/**
|
|
255
|
+
* Compute a stable content hash for deduplication across connector runs.
|
|
256
|
+
* Prefer externalId-based identity; this hash is the fallback.
|
|
257
|
+
*/
|
|
258
|
+
export function computeEvidenceContentHash(content) {
|
|
259
|
+
return computeEvidenceContentHashSync(content);
|
|
260
|
+
}
|
|
261
|
+
import * as crypto from "node:crypto";
|
|
262
|
+
export function computeEvidenceContentHashSync(content) {
|
|
263
|
+
const canonical = [
|
|
264
|
+
content.platformId,
|
|
265
|
+
content.capabilityId,
|
|
266
|
+
content.externalId ?? "",
|
|
267
|
+
content.title ?? "",
|
|
268
|
+
content.summary,
|
|
269
|
+
content.excerpt ?? "",
|
|
270
|
+
content.canonicalText ?? "",
|
|
271
|
+
].join("\u0000");
|
|
272
|
+
return crypto.createHash("sha256").update(canonical).digest("hex").slice(0, 16);
|
|
273
|
+
}
|
|
@@ -11,12 +11,12 @@
|
|
|
11
11
|
*
|
|
12
12
|
* Dependencies:
|
|
13
13
|
* - `src/shared/types/v8-contracts.js` (SourceRef, V8ReasonCode)
|
|
14
|
-
* - `src/storage/v8-state-stores.js` (writeEvidenceItem)
|
|
14
|
+
* - `src/storage/v8-state-stores.js` (writeEvidenceItem, readEvidenceItemById)
|
|
15
15
|
*
|
|
16
16
|
* Boundary:
|
|
17
17
|
* - Does not judge evidence importance.
|
|
18
18
|
* - Does not fabricate evidence on empty or failed connector results.
|
|
19
|
-
* - Deduplicates by
|
|
19
|
+
* - Deduplicates by externalId first, then content hash, across runs.
|
|
20
20
|
*
|
|
21
21
|
* Test coverage: tests/unit/connectors/evidence-normalizer.test.ts
|
|
22
22
|
*/
|
|
@@ -26,7 +26,8 @@ export interface ConnectorReadResult {
|
|
|
26
26
|
status: "success" | "failed" | "unavailable" | "timeout";
|
|
27
27
|
platformId: string;
|
|
28
28
|
capabilityId: string;
|
|
29
|
-
|
|
29
|
+
data?: unknown;
|
|
30
|
+
items?: ConnectorReadItem[];
|
|
30
31
|
observedAt?: string;
|
|
31
32
|
failureReason?: V8ReasonCode;
|
|
32
33
|
}
|
|
@@ -11,21 +11,24 @@
|
|
|
11
11
|
*
|
|
12
12
|
* Dependencies:
|
|
13
13
|
* - `src/shared/types/v8-contracts.js` (SourceRef, V8ReasonCode)
|
|
14
|
-
* - `src/storage/v8-state-stores.js` (writeEvidenceItem)
|
|
14
|
+
* - `src/storage/v8-state-stores.js` (writeEvidenceItem, readEvidenceItemById)
|
|
15
15
|
*
|
|
16
16
|
* Boundary:
|
|
17
17
|
* - Does not judge evidence importance.
|
|
18
18
|
* - Does not fabricate evidence on empty or failed connector results.
|
|
19
|
-
* - Deduplicates by
|
|
19
|
+
* - Deduplicates by externalId first, then content hash, across runs.
|
|
20
20
|
*
|
|
21
21
|
* Test coverage: tests/unit/connectors/evidence-normalizer.test.ts
|
|
22
22
|
*/
|
|
23
23
|
import * as crypto from "node:crypto";
|
|
24
|
-
import {
|
|
24
|
+
import { eq } from "drizzle-orm";
|
|
25
|
+
import { evidenceItem } from "../storage/db/schema/v8-entities.js";
|
|
26
|
+
import { writeEvidenceItem, readEvidenceItemById } from "../storage/v8-state-stores.js";
|
|
27
|
+
import { extractNormalizedEvidenceItems, computeEvidenceContentHashSync, } from "./base/normalized-evidence-content.js";
|
|
25
28
|
// ───────────────────────────────────────────────────────────────
|
|
26
29
|
// Helpers
|
|
27
30
|
// ───────────────────────────────────────────────────────────────
|
|
28
|
-
function
|
|
31
|
+
function computeLegacyContentHash(content) {
|
|
29
32
|
return crypto.createHash("sha256").update(content).digest("hex").slice(0, 16);
|
|
30
33
|
}
|
|
31
34
|
function buildSourceRef(platformId, capabilityId, itemId, observedAt) {
|
|
@@ -37,18 +40,43 @@ function buildSourceRef(platformId, capabilityId, itemId, observedAt) {
|
|
|
37
40
|
resolveStatus: "resolvable",
|
|
38
41
|
};
|
|
39
42
|
}
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
}
|
|
48
|
-
return "public_technical";
|
|
43
|
+
const SECRET_KEYWORD_RE = /\b(?:api[-_]?key|auth[-_]?token|access[-_]?token|secret|password|credential|private[-_]?key)\b/i;
|
|
44
|
+
const SECRET_VALUE_RE = /\b[a-zA-Z0-9_]+\s*[:=]\s*["']([^"'\s]{20,})["']/;
|
|
45
|
+
function inferSensitivityHint(content) {
|
|
46
|
+
// Only flag sensitive when a secret keyword is paired with a value-like shape.
|
|
47
|
+
// Broad keyword matches alone are not enough to classify public_technical.
|
|
48
|
+
if (SECRET_KEYWORD_RE.test(content) && SECRET_VALUE_RE.test(content)) {
|
|
49
|
+
return "sensitive";
|
|
49
50
|
}
|
|
50
51
|
return "public_general";
|
|
51
52
|
}
|
|
53
|
+
function mergeSensitivityHint(fromContent, explicit) {
|
|
54
|
+
if (explicit === "sensitive" || fromContent === "sensitive")
|
|
55
|
+
return "sensitive";
|
|
56
|
+
if (explicit === "private_context" || fromContent === "private_context")
|
|
57
|
+
return "private_context";
|
|
58
|
+
if (explicit === "public_technical" || fromContent === "public_technical")
|
|
59
|
+
return "public_technical";
|
|
60
|
+
return explicit ?? fromContent;
|
|
61
|
+
}
|
|
62
|
+
function stableEvidenceId(platformId, capabilityId, externalId, contentHash) {
|
|
63
|
+
const key = externalId ?? contentHash;
|
|
64
|
+
return `ev_${platformId}_${capabilityId}_${key}`;
|
|
65
|
+
}
|
|
66
|
+
async function findExistingEvidenceByExternalId(db, id) {
|
|
67
|
+
const result = await readEvidenceItemById(db, id);
|
|
68
|
+
if ("row" in result && result.row) {
|
|
69
|
+
return {
|
|
70
|
+
id: result.row.id,
|
|
71
|
+
observedAt: result.row.observedAt,
|
|
72
|
+
payloadJson: result.row.payloadJson ?? null,
|
|
73
|
+
};
|
|
74
|
+
}
|
|
75
|
+
return undefined;
|
|
76
|
+
}
|
|
77
|
+
function buildPayloadJson(content) {
|
|
78
|
+
return JSON.stringify(content);
|
|
79
|
+
}
|
|
52
80
|
// ───────────────────────────────────────────────────────────────
|
|
53
81
|
// Public API
|
|
54
82
|
// ───────────────────────────────────────────────────────────────
|
|
@@ -60,31 +88,109 @@ export async function normalizeConnectorEvidence(db, result, now = new Date().to
|
|
|
60
88
|
emptyReason: "ingestion_connector_failed",
|
|
61
89
|
};
|
|
62
90
|
}
|
|
91
|
+
// Extract content-bearing items from payload
|
|
92
|
+
const observedAt = result.observedAt ?? now;
|
|
93
|
+
const normalizedItems = extractNormalizedEvidenceItems(result.data ?? result.items ?? [], {
|
|
94
|
+
platformId: result.platformId,
|
|
95
|
+
capabilityId: result.capabilityId,
|
|
96
|
+
observedAt,
|
|
97
|
+
summaryProducer: "connector_rules",
|
|
98
|
+
});
|
|
63
99
|
// Empty result — no fabrication
|
|
64
|
-
if (
|
|
100
|
+
if (normalizedItems.length === 0) {
|
|
101
|
+
// Legacy fallback: if caller passed flat items, still try to produce evidence.
|
|
102
|
+
return normalizeLegacyConnectorEvidence(db, result, now);
|
|
103
|
+
}
|
|
104
|
+
// Truncate if over 100 items
|
|
105
|
+
const items = normalizedItems.slice(0, 100);
|
|
106
|
+
const truncated = normalizedItems.length > 100;
|
|
107
|
+
const seenKeys = new Set();
|
|
108
|
+
const evidenceIds = [];
|
|
109
|
+
for (const content of items) {
|
|
110
|
+
const contentHash = computeEvidenceContentHashSync(content);
|
|
111
|
+
const key = `ev_${result.platformId}_${result.capabilityId}_${content.externalId ?? contentHash}`;
|
|
112
|
+
if (seenKeys.has(key))
|
|
113
|
+
continue;
|
|
114
|
+
seenKeys.add(key);
|
|
115
|
+
const evidenceId = stableEvidenceId(result.platformId, result.capabilityId, content.externalId, contentHash);
|
|
116
|
+
const existing = await findExistingEvidenceByExternalId(db, evidenceId);
|
|
117
|
+
const sourceRef = buildSourceRef(result.platformId, result.capabilityId, content.externalId ?? contentHash, observedAt);
|
|
118
|
+
const sensitivityHint = mergeSensitivityHint(inferSensitivityHint(content.summary), inferSensitivityHint(content.title ?? ""));
|
|
119
|
+
if (existing) {
|
|
120
|
+
// Idempotent update: refresh observedAt and seen count; do not duplicate.
|
|
121
|
+
try {
|
|
122
|
+
await db.db
|
|
123
|
+
.update(evidenceItem)
|
|
124
|
+
.set({
|
|
125
|
+
observedAt,
|
|
126
|
+
payloadJson: JSON.stringify({
|
|
127
|
+
...JSON.parse(existing.payloadJson ?? "{}"),
|
|
128
|
+
lastObservedAt: observedAt,
|
|
129
|
+
seenCount: (JSON.parse(existing.payloadJson ?? "{}").seenCount ?? 1) + 1,
|
|
130
|
+
}),
|
|
131
|
+
})
|
|
132
|
+
.where(eq(evidenceItem.id, existing.id));
|
|
133
|
+
evidenceIds.push(existing.id);
|
|
134
|
+
continue;
|
|
135
|
+
}
|
|
136
|
+
catch {
|
|
137
|
+
// Fall through to insert attempt if update fails.
|
|
138
|
+
}
|
|
139
|
+
}
|
|
140
|
+
const writeResult = await writeEvidenceItem(db, {
|
|
141
|
+
id: evidenceId,
|
|
142
|
+
createdAt: now,
|
|
143
|
+
platformId: result.platformId,
|
|
144
|
+
contentHash,
|
|
145
|
+
observedAt,
|
|
146
|
+
sensitivityHint,
|
|
147
|
+
sourceRefs: [sourceRef],
|
|
148
|
+
redactionClass: sensitivityHint === "sensitive" ? "blocked" : "none",
|
|
149
|
+
lifecycleStatus: "pending",
|
|
150
|
+
payloadJson: buildPayloadJson(content),
|
|
151
|
+
});
|
|
152
|
+
if ("id" in writeResult) {
|
|
153
|
+
evidenceIds.push(writeResult.id);
|
|
154
|
+
}
|
|
155
|
+
else {
|
|
156
|
+
// Degraded write — continue with remaining items, report degraded
|
|
157
|
+
return {
|
|
158
|
+
evidenceIds,
|
|
159
|
+
degraded: writeResult,
|
|
160
|
+
};
|
|
161
|
+
}
|
|
162
|
+
}
|
|
163
|
+
return {
|
|
164
|
+
evidenceIds,
|
|
165
|
+
emptyReason: truncated ? "evidence_batch_truncated" : undefined,
|
|
166
|
+
};
|
|
167
|
+
}
|
|
168
|
+
/**
|
|
169
|
+
* Legacy fallback for callers that still pass flat ConnectorReadItem[] without `data`.
|
|
170
|
+
* This preserves the previous behavior while allowing gradual migration to content-bearing payloads.
|
|
171
|
+
*/
|
|
172
|
+
async function normalizeLegacyConnectorEvidence(db, result, now) {
|
|
173
|
+
const items = result.items ?? [];
|
|
174
|
+
if (items.length === 0) {
|
|
65
175
|
return {
|
|
66
176
|
evidenceIds: [],
|
|
67
177
|
emptyReason: "evidence_batch_empty",
|
|
68
178
|
};
|
|
69
179
|
}
|
|
70
|
-
|
|
71
|
-
const items = result.items.slice(0, 100);
|
|
72
|
-
const truncated = result.items.length > 100;
|
|
180
|
+
const truncated = items.length > 100;
|
|
73
181
|
const seenHashes = new Set();
|
|
74
182
|
const evidenceIds = [];
|
|
75
183
|
for (const item of items) {
|
|
76
|
-
if (typeof item.content !== "string")
|
|
184
|
+
if (typeof item.content !== "string")
|
|
77
185
|
continue;
|
|
78
|
-
|
|
79
|
-
const contentHash = computeContentHash(item.content);
|
|
80
|
-
// Deduplicate by content hash
|
|
186
|
+
const contentHash = computeLegacyContentHash(item.content);
|
|
81
187
|
if (seenHashes.has(contentHash))
|
|
82
188
|
continue;
|
|
83
189
|
seenHashes.add(contentHash);
|
|
84
190
|
const itemId = item.id ?? `ev_${contentHash}`;
|
|
85
191
|
const observedAt = result.observedAt ?? now;
|
|
86
192
|
const sourceRef = buildSourceRef(result.platformId, result.capabilityId, itemId, observedAt);
|
|
87
|
-
const sensitivityHint = inferSensitivityHint(item);
|
|
193
|
+
const sensitivityHint = mergeSensitivityHint(inferSensitivityHint(item.content), item.sensitivityHint);
|
|
88
194
|
const writeResult = await writeEvidenceItem(db, {
|
|
89
195
|
id: `ev_${result.platformId}_${itemId}_${observedAt.replace(/[:.]/g, "")}`,
|
|
90
196
|
createdAt: now,
|
|
@@ -101,7 +207,6 @@ export async function normalizeConnectorEvidence(db, result, now = new Date().to
|
|
|
101
207
|
evidenceIds.push(writeResult.id);
|
|
102
208
|
}
|
|
103
209
|
else {
|
|
104
|
-
// Degraded write — continue with remaining items, report degraded
|
|
105
210
|
return {
|
|
106
211
|
evidenceIds,
|
|
107
212
|
degraded: writeResult,
|
|
@@ -10,6 +10,7 @@ import { toCapabilityIntent } from "../orchestrator/effect-dispatcher.js";
|
|
|
10
10
|
import { updateNarrativeAfterEffect } from "../orchestrator/narrative-update.js";
|
|
11
11
|
import { mapLifeEvidence } from "../../../connectors/base/map-life-evidence.js";
|
|
12
12
|
import { appendLifeEvidence } from "../../../storage/life-evidence/append-life-evidence.js";
|
|
13
|
+
import { normalizeConnectorEvidence } from "../../../connectors/evidence-normalizer.js";
|
|
13
14
|
import { recordConnectorAttemptAudit } from "../../../observability/services/audit-closure-recorders.js";
|
|
14
15
|
/**
|
|
15
16
|
* Resolves the heartbeat outcome for a guard-allowed intent (outreach dispatch, quiet orchestration, or default).
|
|
@@ -116,6 +117,20 @@ export async function resolveAllowedIntentResult(intent, runtime, inputs, signal
|
|
|
116
117
|
const errorMessage = err instanceof Error ? err.message : String(err);
|
|
117
118
|
console.warn(`[heartbeat] evidence append failed for ${intent.platformId ?? "unknown"}: ${errorMessage}`);
|
|
118
119
|
}
|
|
120
|
+
// Wave 109 T-CS.R.5: also normalize into v8 EvidenceItem with content-bearing payload.
|
|
121
|
+
try {
|
|
122
|
+
await normalizeConnectorEvidence(deps.state, {
|
|
123
|
+
status: "success",
|
|
124
|
+
platformId: intent.platformId,
|
|
125
|
+
capabilityId: toCapabilityIntent(intent),
|
|
126
|
+
data: result.data,
|
|
127
|
+
observedAt: new Date().toISOString(),
|
|
128
|
+
});
|
|
129
|
+
}
|
|
130
|
+
catch (err) {
|
|
131
|
+
const errorMessage = err instanceof Error ? err.message : String(err);
|
|
132
|
+
console.warn(`[heartbeat] v8 evidence normalization failed for ${intent.platformId ?? "unknown"}: ${errorMessage}`);
|
|
133
|
+
}
|
|
119
134
|
}
|
|
120
135
|
// v7 T-V7C.C.2: record ToolExperience for all connector attempts in heartbeat.
|
|
121
136
|
if (deps.experienceWriter) {
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
/**
|
|
2
2
|
* PerceptionBuilder — Generate PerceptionCard records from EvidenceItem batches.
|
|
3
3
|
*
|
|
4
|
-
* Core logic: Read pending evidence, deduplicate by content hash, build
|
|
4
|
+
* Core logic: Read pending evidence, deduplicate by externalId/content hash, build
|
|
5
5
|
* PerceptionCard with topic, entities, novelty, relevance, summary, risk
|
|
6
6
|
* flags, confidence, and reviewPriority. Rules-only fallback when model
|
|
7
7
|
* assist is unavailable.
|
|
@@ -51,6 +51,8 @@ export interface PerceptionCardResult {
|
|
|
51
51
|
confidence: number;
|
|
52
52
|
evidenceRefs: SourceRef[];
|
|
53
53
|
createdAt: string;
|
|
54
|
+
/** True when the evidence payload lacked readable content and only refs are present. */
|
|
55
|
+
contentMissing?: boolean;
|
|
54
56
|
}
|
|
55
57
|
export interface BuildPerceptionCardsResult {
|
|
56
58
|
status: "completed" | "rules_only" | "blocked" | "empty" | "degraded";
|