@haaaiawd/second-nature 0.2.6 → 0.2.8
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/index.js +9 -0
- package/openclaw.plugin.json +2 -2
- package/package.json +1 -1
- package/runtime/cli/commands/index.js +9 -0
- package/runtime/cli/ops/heartbeat-surface.js +1 -0
- package/runtime/connectors/base/normalized-evidence-content.d.ts +71 -0
- package/runtime/connectors/base/normalized-evidence-content.js +273 -0
- package/runtime/connectors/evidence-normalizer.d.ts +4 -3
- package/runtime/connectors/evidence-normalizer.js +128 -23
- package/runtime/core/second-nature/heartbeat/heartbeat-loop.js +15 -0
- package/runtime/core/second-nature/perception/perception-builder.d.ts +3 -1
- package/runtime/core/second-nature/perception/perception-builder.js +98 -44
- package/runtime/core/second-nature/quiet-dream/daily-rhythm-scheduler.d.ts +6 -3
- package/runtime/core/second-nature/quiet-dream/daily-rhythm-scheduler.js +198 -22
- package/runtime/core/second-nature/quiet-dream/quiet-daily-review-builder.d.ts +19 -3
- package/runtime/core/second-nature/quiet-dream/quiet-daily-review-builder.js +189 -12
- package/runtime/shared/types/v8-contracts.d.ts +1 -1
- package/runtime/storage/db/index.js +2 -0
- package/runtime/storage/db/migrations/v8-003-quiet-closure-refs.js +2 -1
- package/runtime/storage/services/write-validation-gate.d.ts +2 -0
- package/runtime/storage/services/write-validation-gate.js +69 -17
- package/runtime/storage/v8-state-stores.d.ts +25 -0
- package/runtime/storage/v8-state-stores.js +87 -1
package/index.js
CHANGED
|
@@ -108,6 +108,8 @@ const WORKSPACE_BRIDGE_COMMANDS = new Set([
|
|
|
108
108
|
"connector_test",
|
|
109
109
|
"connector_behavior_add",
|
|
110
110
|
"cycle:recent",
|
|
111
|
+
// v8 ops surface (T-ROS.C.1): causal loop health must be reachable from Claw.
|
|
112
|
+
"loop_status",
|
|
111
113
|
// v7 ops surface (T-ROS.C.1 / T-ROS.C.2 / T-ROS.C.3 / T-V7C.C.5): self_health, tool_affordance,
|
|
112
114
|
// heartbeat_digest, snapshot:capture, narrative:diff, timeline, restore, runtime_secret_bootstrap,
|
|
113
115
|
// connector:run, guidance_payload
|
|
@@ -877,6 +879,11 @@ function createHostSafeRouter(spine) {
|
|
|
877
879
|
description: "Show recent cycle summary (workspace runtime required)",
|
|
878
880
|
execute: async () => createUnavailableActionError("HOST_SAFE_CYCLE_RECENT_UNAVAILABLE", "Cycle recent read requires workspace observability database; host-safe plugin does not load persisted audit events.", [], "run_workspace_second_nature_cli_or_full_runtime_package"),
|
|
879
881
|
},
|
|
882
|
+
{
|
|
883
|
+
name: "loop_status",
|
|
884
|
+
description: "Show v8 causal loop health (workspace runtime required)",
|
|
885
|
+
execute: async () => createUnavailableActionError("HOST_SAFE_LOOP_STATUS_UNAVAILABLE", "loop_status requires workspace state database; provide workspaceRoot so the full workspace bridge can read persisted v8 loop state.", ["workspaceRoot"], "reinvoke_with_workspaceRoot_or_set_SECOND_NATURE_WORKSPACE_ROOT"),
|
|
886
|
+
},
|
|
880
887
|
];
|
|
881
888
|
return {
|
|
882
889
|
commands,
|
|
@@ -1088,6 +1095,8 @@ function parseCommandInput(rawArgs) {
|
|
|
1088
1095
|
command,
|
|
1089
1096
|
input: rest[0] ? { limit: Number(rest[0]) } : undefined,
|
|
1090
1097
|
};
|
|
1098
|
+
case "loop_status":
|
|
1099
|
+
return { ok: true, command, input: undefined };
|
|
1091
1100
|
// v7 ops surface (T-ROS.C.2)
|
|
1092
1101
|
case "self_health":
|
|
1093
1102
|
return { ok: true, command, input: undefined };
|
package/openclaw.plugin.json
CHANGED
|
@@ -1,8 +1,8 @@
|
|
|
1
1
|
{
|
|
2
2
|
"id": "second-nature",
|
|
3
3
|
"name": "Second Nature",
|
|
4
|
-
"version": "0.2.
|
|
5
|
-
"description": "OpenClaw native plugin with synchronous surface registration and bundled runtime spine. Set SECOND_NATURE_WORKSPACE_ROOT or tool workspaceRoot to the same path as the agent workspace. Agent inner guide is packaged as agent-inner-guide.md.
|
|
4
|
+
"version": "0.2.8",
|
|
5
|
+
"description": "OpenClaw native plugin with synchronous surface registration and bundled runtime spine. Set SECOND_NATURE_WORKSPACE_ROOT or tool workspaceRoot to the same path as the agent workspace. Agent inner guide is packaged as agent-inner-guide.md. Ops surface: loop_status, self_health, tool_affordance, heartbeat_digest, snapshot:capture, narrative:diff, timeline, restore, runtime_secret_bootstrap, connector:run, guidance_payload.",
|
|
6
6
|
"activation": {
|
|
7
7
|
"onStartup": true,
|
|
8
8
|
"onCapabilities": [
|
package/package.json
CHANGED
|
@@ -449,6 +449,15 @@ export function createCliCommands(deps) {
|
|
|
449
449
|
return surface;
|
|
450
450
|
},
|
|
451
451
|
},
|
|
452
|
+
{
|
|
453
|
+
name: "connector_status",
|
|
454
|
+
description: "T1.2.5 — show connector inventory status, trust, and health conflicts",
|
|
455
|
+
execute: async (input) => {
|
|
456
|
+
const surface = await Promise.resolve(opsRouter.dispatch("connector_status", input));
|
|
457
|
+
flush();
|
|
458
|
+
return surface;
|
|
459
|
+
},
|
|
460
|
+
},
|
|
452
461
|
opsCommand("connector:run", "T-ROS.C.3 — manually execute a connector capability outside heartbeat cadence"),
|
|
453
462
|
opsCommand("loop_status", "T-ROS.C.1 — show v8 causal loop health: stalled stage, next action, and stage summaries"),
|
|
454
463
|
opsCommand("self_health", "T-ROS.C.1 — show v7 self-health snapshot and degraded dimensions"),
|
|
@@ -112,6 +112,7 @@ export async function heartbeatCheck(input) {
|
|
|
112
112
|
else {
|
|
113
113
|
const spine = v8Result;
|
|
114
114
|
surfaceResult.v8Spine = spine;
|
|
115
|
+
surfaceResult.livedExperienceLoopClaimed = Boolean(spine.cycleId && (spine.closureRef || spine.noActionReason));
|
|
115
116
|
surfaceResult.reasons = [
|
|
116
117
|
...surfaceResult.reasons,
|
|
117
118
|
`v8_spine_cycle:${spine.cycleId}`,
|
|
@@ -0,0 +1,71 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* NormalizedEvidenceContent — Cross-platform content-bearing evidence envelope.
|
|
3
|
+
*
|
|
4
|
+
* Core logic: Map arbitrary connector read payloads into a stable, source-backed
|
|
5
|
+
* summary structure that perception, Quiet, and Dream can consume. This is a
|
|
6
|
+
* schema boundary, not a judgment layer.
|
|
7
|
+
*
|
|
8
|
+
* Design authority:
|
|
9
|
+
* - `.anws/v8/04_SYSTEM_DESIGN/connector-system.md`
|
|
10
|
+
* - `.anws/v8/04_SYSTEM_DESIGN/perception-judgment-system.md`
|
|
11
|
+
*
|
|
12
|
+
* Dependencies:
|
|
13
|
+
* - none (pure extraction)
|
|
14
|
+
*
|
|
15
|
+
* Boundary:
|
|
16
|
+
* - Does not classify sensitivity.
|
|
17
|
+
* - Does not redact; callers run redaction separately.
|
|
18
|
+
* - Does not persist; callers write EvidenceItem.
|
|
19
|
+
* - Preserves raw values in canonicalText/excerpt so downstream can decide what to keep.
|
|
20
|
+
*
|
|
21
|
+
* Test coverage: tests/unit/connectors/normalized-evidence-content.test.ts
|
|
22
|
+
*/
|
|
23
|
+
export declare const NORMALIZED_EVIDENCE_SCHEMA_VERSION = 1;
|
|
24
|
+
export type EvidenceSourceKind = "post" | "comment" | "profile" | "task" | "event" | "game_state" | "notification" | "document" | "unknown";
|
|
25
|
+
export type SummaryProducer = "connector_rules" | "model_assist" | "operator_supplied";
|
|
26
|
+
export interface EvidenceActor {
|
|
27
|
+
id?: string;
|
|
28
|
+
displayName?: string;
|
|
29
|
+
role?: string;
|
|
30
|
+
}
|
|
31
|
+
export interface NormalizedEvidenceContent {
|
|
32
|
+
schemaVersion: typeof NORMALIZED_EVIDENCE_SCHEMA_VERSION;
|
|
33
|
+
sourceKind: EvidenceSourceKind;
|
|
34
|
+
platformId: string;
|
|
35
|
+
capabilityId: string;
|
|
36
|
+
externalId?: string;
|
|
37
|
+
title?: string;
|
|
38
|
+
summary: string;
|
|
39
|
+
excerpt?: string;
|
|
40
|
+
canonicalText?: string;
|
|
41
|
+
actor?: EvidenceActor;
|
|
42
|
+
url?: string;
|
|
43
|
+
occurredAt?: string;
|
|
44
|
+
observedAt: string;
|
|
45
|
+
tags?: string[];
|
|
46
|
+
entities?: string[];
|
|
47
|
+
metrics?: Record<string, number | string | boolean>;
|
|
48
|
+
rawContentRef?: string;
|
|
49
|
+
summaryProducer: SummaryProducer;
|
|
50
|
+
}
|
|
51
|
+
export interface ExtractEvidenceOptions {
|
|
52
|
+
platformId: string;
|
|
53
|
+
capabilityId: string;
|
|
54
|
+
observedAt?: string;
|
|
55
|
+
summaryProducer?: SummaryProducer;
|
|
56
|
+
/** Max characters for excerpt. Default 240. */
|
|
57
|
+
excerptMaxChars?: number;
|
|
58
|
+
/** Max characters for canonicalText. Default 2000. */
|
|
59
|
+
canonicalTextMaxChars?: number;
|
|
60
|
+
}
|
|
61
|
+
/**
|
|
62
|
+
* Extract a list of content-bearing evidence items from a connector success payload.
|
|
63
|
+
* Returns empty array when data is not an object/array or contains no extractable items.
|
|
64
|
+
*/
|
|
65
|
+
export declare function extractNormalizedEvidenceItems(data: unknown, options: ExtractEvidenceOptions): NormalizedEvidenceContent[];
|
|
66
|
+
/**
|
|
67
|
+
* Compute a stable content hash for deduplication across connector runs.
|
|
68
|
+
* Prefer externalId-based identity; this hash is the fallback.
|
|
69
|
+
*/
|
|
70
|
+
export declare function computeEvidenceContentHash(content: NormalizedEvidenceContent): string;
|
|
71
|
+
export declare function computeEvidenceContentHashSync(content: NormalizedEvidenceContent): string;
|
|
@@ -0,0 +1,273 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* NormalizedEvidenceContent — Cross-platform content-bearing evidence envelope.
|
|
3
|
+
*
|
|
4
|
+
* Core logic: Map arbitrary connector read payloads into a stable, source-backed
|
|
5
|
+
* summary structure that perception, Quiet, and Dream can consume. This is a
|
|
6
|
+
* schema boundary, not a judgment layer.
|
|
7
|
+
*
|
|
8
|
+
* Design authority:
|
|
9
|
+
* - `.anws/v8/04_SYSTEM_DESIGN/connector-system.md`
|
|
10
|
+
* - `.anws/v8/04_SYSTEM_DESIGN/perception-judgment-system.md`
|
|
11
|
+
*
|
|
12
|
+
* Dependencies:
|
|
13
|
+
* - none (pure extraction)
|
|
14
|
+
*
|
|
15
|
+
* Boundary:
|
|
16
|
+
* - Does not classify sensitivity.
|
|
17
|
+
* - Does not redact; callers run redaction separately.
|
|
18
|
+
* - Does not persist; callers write EvidenceItem.
|
|
19
|
+
* - Preserves raw values in canonicalText/excerpt so downstream can decide what to keep.
|
|
20
|
+
*
|
|
21
|
+
* Test coverage: tests/unit/connectors/normalized-evidence-content.test.ts
|
|
22
|
+
*/
|
|
23
|
+
export const NORMALIZED_EVIDENCE_SCHEMA_VERSION = 1;
|
|
24
|
+
// ───────────────────────────────────────────────────────────────
|
|
25
|
+
// String helpers
|
|
26
|
+
// ───────────────────────────────────────────────────────────────
|
|
27
|
+
function isNonEmptyString(value) {
|
|
28
|
+
return typeof value === "string" && value.trim().length > 0;
|
|
29
|
+
}
|
|
30
|
+
function firstNonEmptyString(values) {
|
|
31
|
+
for (const v of values) {
|
|
32
|
+
if (isNonEmptyString(v))
|
|
33
|
+
return v.trim();
|
|
34
|
+
}
|
|
35
|
+
return undefined;
|
|
36
|
+
}
|
|
37
|
+
function truncate(text, maxChars) {
|
|
38
|
+
if (text.length <= maxChars)
|
|
39
|
+
return text;
|
|
40
|
+
return `${text.slice(0, maxChars)}…`;
|
|
41
|
+
}
|
|
42
|
+
function toStringArray(value) {
|
|
43
|
+
if (!Array.isArray(value))
|
|
44
|
+
return [];
|
|
45
|
+
return value
|
|
46
|
+
.map((item) => (typeof item === "string" ? item.trim() : String(item ?? "").trim()))
|
|
47
|
+
.filter((item) => item.length > 0);
|
|
48
|
+
}
|
|
49
|
+
function normalizeDate(value) {
|
|
50
|
+
if (typeof value !== "string" || value.trim().length === 0)
|
|
51
|
+
return undefined;
|
|
52
|
+
const parsed = Date.parse(value);
|
|
53
|
+
if (Number.isNaN(parsed))
|
|
54
|
+
return undefined;
|
|
55
|
+
return new Date(parsed).toISOString();
|
|
56
|
+
}
|
|
57
|
+
// ───────────────────────────────────────────────────────────────
|
|
58
|
+
// Object traversal helpers
|
|
59
|
+
// ───────────────────────────────────────────────────────────────
|
|
60
|
+
function isRecord(value) {
|
|
61
|
+
return value !== null && typeof value === "object" && !Array.isArray(value);
|
|
62
|
+
}
|
|
63
|
+
function findFirstValue(obj, keys) {
|
|
64
|
+
if (!isRecord(obj))
|
|
65
|
+
return undefined;
|
|
66
|
+
for (const key of keys) {
|
|
67
|
+
if (key in obj)
|
|
68
|
+
return obj[key];
|
|
69
|
+
}
|
|
70
|
+
return undefined;
|
|
71
|
+
}
|
|
72
|
+
function findDeepValue(obj, keys) {
|
|
73
|
+
const queue = [obj];
|
|
74
|
+
while (queue.length > 0) {
|
|
75
|
+
const current = queue.shift();
|
|
76
|
+
if (isRecord(current)) {
|
|
77
|
+
const found = findFirstValue(current, keys);
|
|
78
|
+
if (found !== undefined)
|
|
79
|
+
return found;
|
|
80
|
+
for (const v of Object.values(current)) {
|
|
81
|
+
queue.push(v);
|
|
82
|
+
}
|
|
83
|
+
}
|
|
84
|
+
else if (Array.isArray(current)) {
|
|
85
|
+
for (const v of current) {
|
|
86
|
+
queue.push(v);
|
|
87
|
+
}
|
|
88
|
+
}
|
|
89
|
+
}
|
|
90
|
+
return undefined;
|
|
91
|
+
}
|
|
92
|
+
// ───────────────────────────────────────────────────────────────
|
|
93
|
+
// Source kind inference
|
|
94
|
+
// ───────────────────────────────────────────────────────────────
|
|
95
|
+
function inferSourceKind(item) {
|
|
96
|
+
const typeHint = String(findFirstValue(item, ["type", "kind", "sourceKind", "object", "category"]) ?? "").toLowerCase();
|
|
97
|
+
if (typeHint.includes("post"))
|
|
98
|
+
return "post";
|
|
99
|
+
if (typeHint.includes("comment") || typeHint.includes("reply"))
|
|
100
|
+
return "comment";
|
|
101
|
+
if (typeHint.includes("profile") || typeHint.includes("user") || typeHint.includes("author"))
|
|
102
|
+
return "profile";
|
|
103
|
+
if (typeHint.includes("task") || typeHint.includes("issue") || typeHint.includes("todo"))
|
|
104
|
+
return "task";
|
|
105
|
+
if (typeHint.includes("event") || typeHint.includes("activity"))
|
|
106
|
+
return "event";
|
|
107
|
+
if (typeHint.includes("game") || typeHint.includes("match") || typeHint.includes("score"))
|
|
108
|
+
return "game_state";
|
|
109
|
+
if (typeHint.includes("notification") || typeHint.includes("alert"))
|
|
110
|
+
return "notification";
|
|
111
|
+
if (typeHint.includes("document") || typeHint.includes("article") || typeHint.includes("page"))
|
|
112
|
+
return "document";
|
|
113
|
+
return "unknown";
|
|
114
|
+
}
|
|
115
|
+
// ───────────────────────────────────────────────────────────────
|
|
116
|
+
// Field extraction
|
|
117
|
+
// ───────────────────────────────────────────────────────────────
|
|
118
|
+
function extractId(item) {
|
|
119
|
+
const id = findFirstValue(item, ["id", "externalId", "postId", "taskId", "eventId"]);
|
|
120
|
+
if (id === undefined || id === null)
|
|
121
|
+
return undefined;
|
|
122
|
+
return String(id).trim();
|
|
123
|
+
}
|
|
124
|
+
function extractTitle(item) {
|
|
125
|
+
return firstNonEmptyString([
|
|
126
|
+
findFirstValue(item, ["title", "subject", "heading", "name", "displayName"]),
|
|
127
|
+
]);
|
|
128
|
+
}
|
|
129
|
+
function extractBodyText(item) {
|
|
130
|
+
return firstNonEmptyString([
|
|
131
|
+
findFirstValue(item, ["content", "body", "text", "description", "message", "snippet", "summary"]),
|
|
132
|
+
]);
|
|
133
|
+
}
|
|
134
|
+
function extractUrl(item) {
|
|
135
|
+
const url = findFirstValue(item, ["url", "link", "permalink", "href", "webUrl"]);
|
|
136
|
+
if (!isNonEmptyString(url))
|
|
137
|
+
return undefined;
|
|
138
|
+
try {
|
|
139
|
+
new URL(url);
|
|
140
|
+
return url;
|
|
141
|
+
}
|
|
142
|
+
catch {
|
|
143
|
+
return undefined;
|
|
144
|
+
}
|
|
145
|
+
}
|
|
146
|
+
function extractActor(item) {
|
|
147
|
+
const author = findFirstValue(item, ["author", "creator", "user", "actor", "from"]);
|
|
148
|
+
if (!isRecord(author) && !isNonEmptyString(author))
|
|
149
|
+
return undefined;
|
|
150
|
+
if (isNonEmptyString(author)) {
|
|
151
|
+
return { displayName: author };
|
|
152
|
+
}
|
|
153
|
+
const displayName = firstNonEmptyString([
|
|
154
|
+
findFirstValue(author, ["displayName", "name", "username", "handle", "login"]),
|
|
155
|
+
]);
|
|
156
|
+
const id = firstNonEmptyString([findFirstValue(author, ["id", "userId", "accountId"])]);
|
|
157
|
+
const role = firstNonEmptyString([findFirstValue(author, ["role", "type"])]);
|
|
158
|
+
if (!displayName && !id)
|
|
159
|
+
return undefined;
|
|
160
|
+
return { displayName, id, role };
|
|
161
|
+
}
|
|
162
|
+
function extractOccurredAt(item) {
|
|
163
|
+
return normalizeDate(findFirstValue(item, ["createdAt", "publishedAt", "occurredAt", "timestamp", "date", "time"]));
|
|
164
|
+
}
|
|
165
|
+
function extractTags(item) {
|
|
166
|
+
const tags = findFirstValue(item, ["tags", "labels", "categories", "topics"]);
|
|
167
|
+
return toStringArray(tags);
|
|
168
|
+
}
|
|
169
|
+
function extractEntities(item) {
|
|
170
|
+
const entities = findFirstValue(item, ["entities", "mentions", "hashtags", "keywords"]);
|
|
171
|
+
return toStringArray(entities);
|
|
172
|
+
}
|
|
173
|
+
function extractMetrics(item) {
|
|
174
|
+
const metrics = findFirstValue(item, ["metrics", "stats", "counts", "engagement"]);
|
|
175
|
+
if (!isRecord(metrics))
|
|
176
|
+
return undefined;
|
|
177
|
+
const out = {};
|
|
178
|
+
for (const [k, v] of Object.entries(metrics)) {
|
|
179
|
+
if (typeof v === "number" || typeof v === "string" || typeof v === "boolean") {
|
|
180
|
+
out[k] = v;
|
|
181
|
+
}
|
|
182
|
+
}
|
|
183
|
+
return Object.keys(out).length > 0 ? out : undefined;
|
|
184
|
+
}
|
|
185
|
+
// ───────────────────────────────────────────────────────────────
|
|
186
|
+
// Item extraction from connector payload
|
|
187
|
+
// ───────────────────────────────────────────────────────────────
|
|
188
|
+
function extractItems(data) {
|
|
189
|
+
if (Array.isArray(data))
|
|
190
|
+
return data;
|
|
191
|
+
if (!isRecord(data))
|
|
192
|
+
return [];
|
|
193
|
+
for (const key of ["items", "data", "results", "posts", "nodes", "agents", "edges", "entries", "feed"]) {
|
|
194
|
+
const candidate = data[key];
|
|
195
|
+
if (Array.isArray(candidate))
|
|
196
|
+
return candidate;
|
|
197
|
+
}
|
|
198
|
+
// If the payload itself looks like a single item, treat it as one-item array.
|
|
199
|
+
if ("id" in data || "title" in data || "content" in data)
|
|
200
|
+
return [data];
|
|
201
|
+
return [];
|
|
202
|
+
}
|
|
203
|
+
function normalizeSingleItem(item, options) {
|
|
204
|
+
if (!isRecord(item))
|
|
205
|
+
return null;
|
|
206
|
+
const sourceKind = inferSourceKind(item);
|
|
207
|
+
const externalId = extractId(item);
|
|
208
|
+
const title = extractTitle(item);
|
|
209
|
+
const bodyText = extractBodyText(item);
|
|
210
|
+
const url = extractUrl(item);
|
|
211
|
+
const actor = extractActor(item);
|
|
212
|
+
const occurredAt = extractOccurredAt(item);
|
|
213
|
+
const tags = extractTags(item);
|
|
214
|
+
const entities = extractEntities(item);
|
|
215
|
+
const metrics = extractMetrics(item);
|
|
216
|
+
const rawText = bodyText ?? title ?? "[no readable content]";
|
|
217
|
+
const summary = truncate(rawText, 160);
|
|
218
|
+
const excerpt = rawText.length > summary.length ? truncate(rawText, options.excerptMaxChars ?? 240) : undefined;
|
|
219
|
+
const canonicalText = truncate(rawText, options.canonicalTextMaxChars ?? 2000);
|
|
220
|
+
return {
|
|
221
|
+
schemaVersion: NORMALIZED_EVIDENCE_SCHEMA_VERSION,
|
|
222
|
+
sourceKind,
|
|
223
|
+
platformId: options.platformId,
|
|
224
|
+
capabilityId: options.capabilityId,
|
|
225
|
+
externalId,
|
|
226
|
+
title,
|
|
227
|
+
summary,
|
|
228
|
+
excerpt,
|
|
229
|
+
canonicalText,
|
|
230
|
+
actor,
|
|
231
|
+
url,
|
|
232
|
+
occurredAt,
|
|
233
|
+
observedAt: options.observedAt ?? new Date().toISOString(),
|
|
234
|
+
tags,
|
|
235
|
+
entities,
|
|
236
|
+
metrics,
|
|
237
|
+
summaryProducer: options.summaryProducer ?? "connector_rules",
|
|
238
|
+
};
|
|
239
|
+
}
|
|
240
|
+
/**
|
|
241
|
+
* Extract a list of content-bearing evidence items from a connector success payload.
|
|
242
|
+
* Returns empty array when data is not an object/array or contains no extractable items.
|
|
243
|
+
*/
|
|
244
|
+
export function extractNormalizedEvidenceItems(data, options) {
|
|
245
|
+
const items = extractItems(data);
|
|
246
|
+
const out = [];
|
|
247
|
+
for (const item of items) {
|
|
248
|
+
const normalized = normalizeSingleItem(item, options);
|
|
249
|
+
if (normalized)
|
|
250
|
+
out.push(normalized);
|
|
251
|
+
}
|
|
252
|
+
return out;
|
|
253
|
+
}
|
|
254
|
+
/**
|
|
255
|
+
* Compute a stable content hash for deduplication across connector runs.
|
|
256
|
+
* Prefer externalId-based identity; this hash is the fallback.
|
|
257
|
+
*/
|
|
258
|
+
export function computeEvidenceContentHash(content) {
|
|
259
|
+
return computeEvidenceContentHashSync(content);
|
|
260
|
+
}
|
|
261
|
+
import * as crypto from "node:crypto";
|
|
262
|
+
export function computeEvidenceContentHashSync(content) {
|
|
263
|
+
const canonical = [
|
|
264
|
+
content.platformId,
|
|
265
|
+
content.capabilityId,
|
|
266
|
+
content.externalId ?? "",
|
|
267
|
+
content.title ?? "",
|
|
268
|
+
content.summary,
|
|
269
|
+
content.excerpt ?? "",
|
|
270
|
+
content.canonicalText ?? "",
|
|
271
|
+
].join("\u0000");
|
|
272
|
+
return crypto.createHash("sha256").update(canonical).digest("hex").slice(0, 16);
|
|
273
|
+
}
|
|
@@ -11,12 +11,12 @@
|
|
|
11
11
|
*
|
|
12
12
|
* Dependencies:
|
|
13
13
|
* - `src/shared/types/v8-contracts.js` (SourceRef, V8ReasonCode)
|
|
14
|
-
* - `src/storage/v8-state-stores.js` (writeEvidenceItem)
|
|
14
|
+
* - `src/storage/v8-state-stores.js` (writeEvidenceItem, readEvidenceItemById)
|
|
15
15
|
*
|
|
16
16
|
* Boundary:
|
|
17
17
|
* - Does not judge evidence importance.
|
|
18
18
|
* - Does not fabricate evidence on empty or failed connector results.
|
|
19
|
-
* - Deduplicates by
|
|
19
|
+
* - Deduplicates by externalId first, then content hash, across runs.
|
|
20
20
|
*
|
|
21
21
|
* Test coverage: tests/unit/connectors/evidence-normalizer.test.ts
|
|
22
22
|
*/
|
|
@@ -26,7 +26,8 @@ export interface ConnectorReadResult {
|
|
|
26
26
|
status: "success" | "failed" | "unavailable" | "timeout";
|
|
27
27
|
platformId: string;
|
|
28
28
|
capabilityId: string;
|
|
29
|
-
|
|
29
|
+
data?: unknown;
|
|
30
|
+
items?: ConnectorReadItem[];
|
|
30
31
|
observedAt?: string;
|
|
31
32
|
failureReason?: V8ReasonCode;
|
|
32
33
|
}
|
|
@@ -11,21 +11,24 @@
|
|
|
11
11
|
*
|
|
12
12
|
* Dependencies:
|
|
13
13
|
* - `src/shared/types/v8-contracts.js` (SourceRef, V8ReasonCode)
|
|
14
|
-
* - `src/storage/v8-state-stores.js` (writeEvidenceItem)
|
|
14
|
+
* - `src/storage/v8-state-stores.js` (writeEvidenceItem, readEvidenceItemById)
|
|
15
15
|
*
|
|
16
16
|
* Boundary:
|
|
17
17
|
* - Does not judge evidence importance.
|
|
18
18
|
* - Does not fabricate evidence on empty or failed connector results.
|
|
19
|
-
* - Deduplicates by
|
|
19
|
+
* - Deduplicates by externalId first, then content hash, across runs.
|
|
20
20
|
*
|
|
21
21
|
* Test coverage: tests/unit/connectors/evidence-normalizer.test.ts
|
|
22
22
|
*/
|
|
23
23
|
import * as crypto from "node:crypto";
|
|
24
|
-
import {
|
|
24
|
+
import { eq } from "drizzle-orm";
|
|
25
|
+
import { evidenceItem } from "../storage/db/schema/v8-entities.js";
|
|
26
|
+
import { writeEvidenceItem, readEvidenceItemById } from "../storage/v8-state-stores.js";
|
|
27
|
+
import { extractNormalizedEvidenceItems, computeEvidenceContentHashSync, } from "./base/normalized-evidence-content.js";
|
|
25
28
|
// ───────────────────────────────────────────────────────────────
|
|
26
29
|
// Helpers
|
|
27
30
|
// ───────────────────────────────────────────────────────────────
|
|
28
|
-
function
|
|
31
|
+
function computeLegacyContentHash(content) {
|
|
29
32
|
return crypto.createHash("sha256").update(content).digest("hex").slice(0, 16);
|
|
30
33
|
}
|
|
31
34
|
function buildSourceRef(platformId, capabilityId, itemId, observedAt) {
|
|
@@ -37,18 +40,43 @@ function buildSourceRef(platformId, capabilityId, itemId, observedAt) {
|
|
|
37
40
|
resolveStatus: "resolvable",
|
|
38
41
|
};
|
|
39
42
|
}
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
}
|
|
48
|
-
return "public_technical";
|
|
43
|
+
const SECRET_KEYWORD_RE = /\b(?:api[-_]?key|auth[-_]?token|access[-_]?token|secret|password|credential|private[-_]?key)\b/i;
|
|
44
|
+
const SECRET_VALUE_RE = /\b[a-zA-Z0-9_]+\s*[:=]\s*["']([^"'\s]{20,})["']/;
|
|
45
|
+
function inferSensitivityHint(content) {
|
|
46
|
+
// Only flag sensitive when a secret keyword is paired with a value-like shape.
|
|
47
|
+
// Broad keyword matches alone are not enough to classify public_technical.
|
|
48
|
+
if (SECRET_KEYWORD_RE.test(content) && SECRET_VALUE_RE.test(content)) {
|
|
49
|
+
return "sensitive";
|
|
49
50
|
}
|
|
50
51
|
return "public_general";
|
|
51
52
|
}
|
|
53
|
+
function mergeSensitivityHint(fromContent, explicit) {
|
|
54
|
+
if (explicit === "sensitive" || fromContent === "sensitive")
|
|
55
|
+
return "sensitive";
|
|
56
|
+
if (explicit === "private_context" || fromContent === "private_context")
|
|
57
|
+
return "private_context";
|
|
58
|
+
if (explicit === "public_technical" || fromContent === "public_technical")
|
|
59
|
+
return "public_technical";
|
|
60
|
+
return explicit ?? fromContent;
|
|
61
|
+
}
|
|
62
|
+
function stableEvidenceId(platformId, capabilityId, externalId, contentHash) {
|
|
63
|
+
const key = externalId ?? contentHash;
|
|
64
|
+
return `ev_${platformId}_${capabilityId}_${key}`;
|
|
65
|
+
}
|
|
66
|
+
async function findExistingEvidenceByExternalId(db, id) {
|
|
67
|
+
const result = await readEvidenceItemById(db, id);
|
|
68
|
+
if ("row" in result && result.row) {
|
|
69
|
+
return {
|
|
70
|
+
id: result.row.id,
|
|
71
|
+
observedAt: result.row.observedAt,
|
|
72
|
+
payloadJson: result.row.payloadJson ?? null,
|
|
73
|
+
};
|
|
74
|
+
}
|
|
75
|
+
return undefined;
|
|
76
|
+
}
|
|
77
|
+
function buildPayloadJson(content) {
|
|
78
|
+
return JSON.stringify(content);
|
|
79
|
+
}
|
|
52
80
|
// ───────────────────────────────────────────────────────────────
|
|
53
81
|
// Public API
|
|
54
82
|
// ───────────────────────────────────────────────────────────────
|
|
@@ -60,31 +88,109 @@ export async function normalizeConnectorEvidence(db, result, now = new Date().to
|
|
|
60
88
|
emptyReason: "ingestion_connector_failed",
|
|
61
89
|
};
|
|
62
90
|
}
|
|
91
|
+
// Extract content-bearing items from payload
|
|
92
|
+
const observedAt = result.observedAt ?? now;
|
|
93
|
+
const normalizedItems = extractNormalizedEvidenceItems(result.data ?? result.items ?? [], {
|
|
94
|
+
platformId: result.platformId,
|
|
95
|
+
capabilityId: result.capabilityId,
|
|
96
|
+
observedAt,
|
|
97
|
+
summaryProducer: "connector_rules",
|
|
98
|
+
});
|
|
63
99
|
// Empty result — no fabrication
|
|
64
|
-
if (
|
|
100
|
+
if (normalizedItems.length === 0) {
|
|
101
|
+
// Legacy fallback: if caller passed flat items, still try to produce evidence.
|
|
102
|
+
return normalizeLegacyConnectorEvidence(db, result, now);
|
|
103
|
+
}
|
|
104
|
+
// Truncate if over 100 items
|
|
105
|
+
const items = normalizedItems.slice(0, 100);
|
|
106
|
+
const truncated = normalizedItems.length > 100;
|
|
107
|
+
const seenKeys = new Set();
|
|
108
|
+
const evidenceIds = [];
|
|
109
|
+
for (const content of items) {
|
|
110
|
+
const contentHash = computeEvidenceContentHashSync(content);
|
|
111
|
+
const key = `ev_${result.platformId}_${result.capabilityId}_${content.externalId ?? contentHash}`;
|
|
112
|
+
if (seenKeys.has(key))
|
|
113
|
+
continue;
|
|
114
|
+
seenKeys.add(key);
|
|
115
|
+
const evidenceId = stableEvidenceId(result.platformId, result.capabilityId, content.externalId, contentHash);
|
|
116
|
+
const existing = await findExistingEvidenceByExternalId(db, evidenceId);
|
|
117
|
+
const sourceRef = buildSourceRef(result.platformId, result.capabilityId, content.externalId ?? contentHash, observedAt);
|
|
118
|
+
const sensitivityHint = mergeSensitivityHint(inferSensitivityHint(content.summary), inferSensitivityHint(content.title ?? ""));
|
|
119
|
+
if (existing) {
|
|
120
|
+
// Idempotent update: refresh observedAt and seen count; do not duplicate.
|
|
121
|
+
try {
|
|
122
|
+
await db.db
|
|
123
|
+
.update(evidenceItem)
|
|
124
|
+
.set({
|
|
125
|
+
observedAt,
|
|
126
|
+
payloadJson: JSON.stringify({
|
|
127
|
+
...JSON.parse(existing.payloadJson ?? "{}"),
|
|
128
|
+
lastObservedAt: observedAt,
|
|
129
|
+
seenCount: (JSON.parse(existing.payloadJson ?? "{}").seenCount ?? 1) + 1,
|
|
130
|
+
}),
|
|
131
|
+
})
|
|
132
|
+
.where(eq(evidenceItem.id, existing.id));
|
|
133
|
+
evidenceIds.push(existing.id);
|
|
134
|
+
continue;
|
|
135
|
+
}
|
|
136
|
+
catch {
|
|
137
|
+
// Fall through to insert attempt if update fails.
|
|
138
|
+
}
|
|
139
|
+
}
|
|
140
|
+
const writeResult = await writeEvidenceItem(db, {
|
|
141
|
+
id: evidenceId,
|
|
142
|
+
createdAt: now,
|
|
143
|
+
platformId: result.platformId,
|
|
144
|
+
contentHash,
|
|
145
|
+
observedAt,
|
|
146
|
+
sensitivityHint,
|
|
147
|
+
sourceRefs: [sourceRef],
|
|
148
|
+
redactionClass: sensitivityHint === "sensitive" ? "blocked" : "none",
|
|
149
|
+
lifecycleStatus: "pending",
|
|
150
|
+
payloadJson: buildPayloadJson(content),
|
|
151
|
+
});
|
|
152
|
+
if ("id" in writeResult) {
|
|
153
|
+
evidenceIds.push(writeResult.id);
|
|
154
|
+
}
|
|
155
|
+
else {
|
|
156
|
+
// Degraded write — continue with remaining items, report degraded
|
|
157
|
+
return {
|
|
158
|
+
evidenceIds,
|
|
159
|
+
degraded: writeResult,
|
|
160
|
+
};
|
|
161
|
+
}
|
|
162
|
+
}
|
|
163
|
+
return {
|
|
164
|
+
evidenceIds,
|
|
165
|
+
emptyReason: truncated ? "evidence_batch_truncated" : undefined,
|
|
166
|
+
};
|
|
167
|
+
}
|
|
168
|
+
/**
|
|
169
|
+
* Legacy fallback for callers that still pass flat ConnectorReadItem[] without `data`.
|
|
170
|
+
* This preserves the previous behavior while allowing gradual migration to content-bearing payloads.
|
|
171
|
+
*/
|
|
172
|
+
async function normalizeLegacyConnectorEvidence(db, result, now) {
|
|
173
|
+
const items = result.items ?? [];
|
|
174
|
+
if (items.length === 0) {
|
|
65
175
|
return {
|
|
66
176
|
evidenceIds: [],
|
|
67
177
|
emptyReason: "evidence_batch_empty",
|
|
68
178
|
};
|
|
69
179
|
}
|
|
70
|
-
|
|
71
|
-
const items = result.items.slice(0, 100);
|
|
72
|
-
const truncated = result.items.length > 100;
|
|
180
|
+
const truncated = items.length > 100;
|
|
73
181
|
const seenHashes = new Set();
|
|
74
182
|
const evidenceIds = [];
|
|
75
183
|
for (const item of items) {
|
|
76
|
-
if (typeof item.content !== "string")
|
|
184
|
+
if (typeof item.content !== "string")
|
|
77
185
|
continue;
|
|
78
|
-
|
|
79
|
-
const contentHash = computeContentHash(item.content);
|
|
80
|
-
// Deduplicate by content hash
|
|
186
|
+
const contentHash = computeLegacyContentHash(item.content);
|
|
81
187
|
if (seenHashes.has(contentHash))
|
|
82
188
|
continue;
|
|
83
189
|
seenHashes.add(contentHash);
|
|
84
190
|
const itemId = item.id ?? `ev_${contentHash}`;
|
|
85
191
|
const observedAt = result.observedAt ?? now;
|
|
86
192
|
const sourceRef = buildSourceRef(result.platformId, result.capabilityId, itemId, observedAt);
|
|
87
|
-
const sensitivityHint = inferSensitivityHint(item);
|
|
193
|
+
const sensitivityHint = mergeSensitivityHint(inferSensitivityHint(item.content), item.sensitivityHint);
|
|
88
194
|
const writeResult = await writeEvidenceItem(db, {
|
|
89
195
|
id: `ev_${result.platformId}_${itemId}_${observedAt.replace(/[:.]/g, "")}`,
|
|
90
196
|
createdAt: now,
|
|
@@ -101,7 +207,6 @@ export async function normalizeConnectorEvidence(db, result, now = new Date().to
|
|
|
101
207
|
evidenceIds.push(writeResult.id);
|
|
102
208
|
}
|
|
103
209
|
else {
|
|
104
|
-
// Degraded write — continue with remaining items, report degraded
|
|
105
210
|
return {
|
|
106
211
|
evidenceIds,
|
|
107
212
|
degraded: writeResult,
|