engrm 0.4.38 → 0.4.39
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +53 -2
- package/dist/cli.js +446 -62
- package/dist/hooks/elicitation-result.js +57 -1
- package/dist/hooks/post-tool-use.js +57 -1
- package/dist/hooks/pre-compact.js +57 -1
- package/dist/hooks/sentinel.js +20 -0
- package/dist/hooks/session-start.js +82 -12
- package/dist/hooks/stop.js +305 -186
- package/dist/hooks/user-prompt-submit.js +57 -1
- package/dist/server.js +339 -76
- package/opencode/README.md +70 -0
- package/opencode/install-or-update-opencode-plugin.sh +46 -0
- package/opencode/opencode.example.json +14 -0
- package/opencode/plugin/engrm-opencode.js +61 -0
- package/package.json +10 -4
package/dist/hooks/stop.js
CHANGED
|
@@ -389,6 +389,16 @@ function createDefaultConfig() {
|
|
|
389
389
|
},
|
|
390
390
|
transcript_analysis: {
|
|
391
391
|
enabled: false
|
|
392
|
+
},
|
|
393
|
+
http: {
|
|
394
|
+
enabled: false,
|
|
395
|
+
port: 3767,
|
|
396
|
+
bearer_tokens: []
|
|
397
|
+
},
|
|
398
|
+
fleet: {
|
|
399
|
+
project_name: "shared-experience",
|
|
400
|
+
namespace: "",
|
|
401
|
+
api_key: ""
|
|
392
402
|
}
|
|
393
403
|
};
|
|
394
404
|
}
|
|
@@ -450,6 +460,16 @@ function loadConfig() {
|
|
|
450
460
|
},
|
|
451
461
|
transcript_analysis: {
|
|
452
462
|
enabled: asBool(config["transcript_analysis"]?.["enabled"], defaults.transcript_analysis.enabled)
|
|
463
|
+
},
|
|
464
|
+
http: {
|
|
465
|
+
enabled: asBool(config["http"]?.["enabled"], defaults.http.enabled),
|
|
466
|
+
port: asNumber(config["http"]?.["port"], defaults.http.port),
|
|
467
|
+
bearer_tokens: asStringArray(config["http"]?.["bearer_tokens"], defaults.http.bearer_tokens)
|
|
468
|
+
},
|
|
469
|
+
fleet: {
|
|
470
|
+
project_name: asString(config["fleet"]?.["project_name"], defaults.fleet.project_name),
|
|
471
|
+
namespace: asString(config["fleet"]?.["namespace"], defaults.fleet.namespace),
|
|
472
|
+
api_key: asString(config["fleet"]?.["api_key"], defaults.fleet.api_key)
|
|
453
473
|
}
|
|
454
474
|
};
|
|
455
475
|
}
|
|
@@ -2251,16 +2271,16 @@ class VectorClient {
|
|
|
2251
2271
|
apiKey;
|
|
2252
2272
|
siteId;
|
|
2253
2273
|
namespace;
|
|
2254
|
-
constructor(config) {
|
|
2274
|
+
constructor(config, overrides = {}) {
|
|
2255
2275
|
const baseUrl = getBaseUrl(config);
|
|
2256
|
-
const apiKey = getApiKey(config);
|
|
2276
|
+
const apiKey = overrides.apiKey ?? getApiKey(config);
|
|
2257
2277
|
if (!baseUrl || !apiKey) {
|
|
2258
2278
|
throw new Error("VectorClient requires candengo_url and candengo_api_key");
|
|
2259
2279
|
}
|
|
2260
2280
|
this.baseUrl = baseUrl.replace(/\/$/, "");
|
|
2261
2281
|
this.apiKey = apiKey;
|
|
2262
|
-
this.siteId = config.site_id;
|
|
2263
|
-
this.namespace = config.namespace;
|
|
2282
|
+
this.siteId = overrides.siteId ?? config.site_id;
|
|
2283
|
+
this.namespace = overrides.namespace ?? config.namespace;
|
|
2264
2284
|
}
|
|
2265
2285
|
static isConfigured(config) {
|
|
2266
2286
|
return getApiKey(config) !== null && getBaseUrl(config) !== null;
|
|
@@ -2476,11 +2496,179 @@ function parseJsonArray2(value) {
|
|
|
2476
2496
|
}
|
|
2477
2497
|
}
|
|
2478
2498
|
|
|
2479
|
-
// src/
|
|
2480
|
-
|
|
2499
|
+
// src/capture/scrubber.ts
|
|
2500
|
+
var DEFAULT_PATTERNS = [
|
|
2501
|
+
{
|
|
2502
|
+
source: "sk-[a-zA-Z0-9]{20,}",
|
|
2503
|
+
flags: "g",
|
|
2504
|
+
replacement: "[REDACTED_API_KEY]",
|
|
2505
|
+
description: "OpenAI API keys",
|
|
2506
|
+
category: "api_key",
|
|
2507
|
+
severity: "critical"
|
|
2508
|
+
},
|
|
2509
|
+
{
|
|
2510
|
+
source: "Bearer [a-zA-Z0-9\\-._~+/]+=*",
|
|
2511
|
+
flags: "g",
|
|
2512
|
+
replacement: "[REDACTED_BEARER]",
|
|
2513
|
+
description: "Bearer auth tokens",
|
|
2514
|
+
category: "token",
|
|
2515
|
+
severity: "medium"
|
|
2516
|
+
},
|
|
2517
|
+
{
|
|
2518
|
+
source: "password[=:]\\s*\\S+",
|
|
2519
|
+
flags: "gi",
|
|
2520
|
+
replacement: "password=[REDACTED]",
|
|
2521
|
+
description: "Passwords in config",
|
|
2522
|
+
category: "password",
|
|
2523
|
+
severity: "high"
|
|
2524
|
+
},
|
|
2525
|
+
{
|
|
2526
|
+
source: "postgresql://[^\\s]+",
|
|
2527
|
+
flags: "g",
|
|
2528
|
+
replacement: "[REDACTED_DB_URL]",
|
|
2529
|
+
description: "PostgreSQL connection strings",
|
|
2530
|
+
category: "db_url",
|
|
2531
|
+
severity: "high"
|
|
2532
|
+
},
|
|
2533
|
+
{
|
|
2534
|
+
source: "mongodb://[^\\s]+",
|
|
2535
|
+
flags: "g",
|
|
2536
|
+
replacement: "[REDACTED_DB_URL]",
|
|
2537
|
+
description: "MongoDB connection strings",
|
|
2538
|
+
category: "db_url",
|
|
2539
|
+
severity: "high"
|
|
2540
|
+
},
|
|
2541
|
+
{
|
|
2542
|
+
source: "mysql://[^\\s]+",
|
|
2543
|
+
flags: "g",
|
|
2544
|
+
replacement: "[REDACTED_DB_URL]",
|
|
2545
|
+
description: "MySQL connection strings",
|
|
2546
|
+
category: "db_url",
|
|
2547
|
+
severity: "high"
|
|
2548
|
+
},
|
|
2549
|
+
{
|
|
2550
|
+
source: "AKIA[A-Z0-9]{16}",
|
|
2551
|
+
flags: "g",
|
|
2552
|
+
replacement: "[REDACTED_AWS_KEY]",
|
|
2553
|
+
description: "AWS access keys",
|
|
2554
|
+
category: "api_key",
|
|
2555
|
+
severity: "critical"
|
|
2556
|
+
},
|
|
2557
|
+
{
|
|
2558
|
+
source: "ghp_[a-zA-Z0-9]{36}",
|
|
2559
|
+
flags: "g",
|
|
2560
|
+
replacement: "[REDACTED_GH_TOKEN]",
|
|
2561
|
+
description: "GitHub personal access tokens",
|
|
2562
|
+
category: "token",
|
|
2563
|
+
severity: "high"
|
|
2564
|
+
},
|
|
2565
|
+
{
|
|
2566
|
+
source: "gho_[a-zA-Z0-9]{36}",
|
|
2567
|
+
flags: "g",
|
|
2568
|
+
replacement: "[REDACTED_GH_TOKEN]",
|
|
2569
|
+
description: "GitHub OAuth tokens",
|
|
2570
|
+
category: "token",
|
|
2571
|
+
severity: "high"
|
|
2572
|
+
},
|
|
2573
|
+
{
|
|
2574
|
+
source: "github_pat_[a-zA-Z0-9_]{22,}",
|
|
2575
|
+
flags: "g",
|
|
2576
|
+
replacement: "[REDACTED_GH_TOKEN]",
|
|
2577
|
+
description: "GitHub fine-grained PATs",
|
|
2578
|
+
category: "token",
|
|
2579
|
+
severity: "high"
|
|
2580
|
+
},
|
|
2581
|
+
{
|
|
2582
|
+
source: "cvk_[a-f0-9]{64}",
|
|
2583
|
+
flags: "g",
|
|
2584
|
+
replacement: "[REDACTED_CANDENGO_KEY]",
|
|
2585
|
+
description: "Candengo API keys",
|
|
2586
|
+
category: "api_key",
|
|
2587
|
+
severity: "critical"
|
|
2588
|
+
},
|
|
2589
|
+
{
|
|
2590
|
+
source: "xox[bpras]-[a-zA-Z0-9\\-]+",
|
|
2591
|
+
flags: "g",
|
|
2592
|
+
replacement: "[REDACTED_SLACK_TOKEN]",
|
|
2593
|
+
description: "Slack tokens",
|
|
2594
|
+
category: "token",
|
|
2595
|
+
severity: "high"
|
|
2596
|
+
}
|
|
2597
|
+
];
|
|
2598
|
+
function compileCustomPatterns(patterns) {
|
|
2599
|
+
const compiled = [];
|
|
2600
|
+
for (const pattern of patterns) {
|
|
2601
|
+
try {
|
|
2602
|
+
new RegExp(pattern);
|
|
2603
|
+
compiled.push({
|
|
2604
|
+
source: pattern,
|
|
2605
|
+
flags: "g",
|
|
2606
|
+
replacement: "[REDACTED_CUSTOM]",
|
|
2607
|
+
description: `Custom pattern: ${pattern}`,
|
|
2608
|
+
category: "custom",
|
|
2609
|
+
severity: "medium"
|
|
2610
|
+
});
|
|
2611
|
+
} catch {}
|
|
2612
|
+
}
|
|
2613
|
+
return compiled;
|
|
2614
|
+
}
|
|
2615
|
+
function scrubSecrets(text, customPatterns = []) {
|
|
2616
|
+
let result = text;
|
|
2617
|
+
const allPatterns = [...DEFAULT_PATTERNS, ...compileCustomPatterns(customPatterns)];
|
|
2618
|
+
for (const pattern of allPatterns) {
|
|
2619
|
+
result = result.replace(new RegExp(pattern.source, pattern.flags), pattern.replacement);
|
|
2620
|
+
}
|
|
2621
|
+
return result;
|
|
2622
|
+
}
|
|
2623
|
+
function containsSecrets(text, customPatterns = []) {
|
|
2624
|
+
const allPatterns = [...DEFAULT_PATTERNS, ...compileCustomPatterns(customPatterns)];
|
|
2625
|
+
for (const pattern of allPatterns) {
|
|
2626
|
+
if (new RegExp(pattern.source, pattern.flags).test(text))
|
|
2627
|
+
return true;
|
|
2628
|
+
}
|
|
2629
|
+
return false;
|
|
2630
|
+
}
|
|
2631
|
+
var FLEET_HOSTNAME_PATTERN = /\b(?=.{1,253}\b)(?!-)(?:[a-z0-9](?:[a-z0-9-]{0,61}[a-z0-9])?\.)+[a-z]{2,63}\b/gi;
|
|
2632
|
+
var FLEET_IP_PATTERN = /\b(?:\d{1,3}\.){3}\d{1,3}\b/g;
|
|
2633
|
+
var FLEET_MAC_PATTERN = /\b(?:[0-9a-f]{2}[:-]){5}[0-9a-f]{2}\b/gi;
|
|
2634
|
+
function scrubFleetIdentifiers(text) {
|
|
2635
|
+
return text.replace(FLEET_MAC_PATTERN, "[REDACTED_MAC]").replace(FLEET_IP_PATTERN, "[REDACTED_IP]").replace(FLEET_HOSTNAME_PATTERN, "[REDACTED_HOSTNAME]");
|
|
2636
|
+
}
|
|
2637
|
+
|
|
2638
|
+
// src/sync/targets.ts
|
|
2639
|
+
function isFleetProjectName(projectName, config) {
|
|
2640
|
+
const fleetProjectName = config.fleet?.project_name ?? "shared-experience";
|
|
2641
|
+
if (!projectName || !fleetProjectName)
|
|
2642
|
+
return false;
|
|
2643
|
+
return projectName.trim().toLowerCase() === fleetProjectName.trim().toLowerCase();
|
|
2644
|
+
}
|
|
2645
|
+
function hasFleetTarget(config) {
|
|
2646
|
+
return Boolean(config.fleet?.namespace?.trim() && config.fleet?.api_key?.trim() && (config.fleet?.project_name ?? "shared-experience").trim());
|
|
2647
|
+
}
|
|
2648
|
+
function resolveSyncTarget(config, projectName) {
|
|
2649
|
+
if (isFleetProjectName(projectName, config) && hasFleetTarget(config)) {
|
|
2650
|
+
return {
|
|
2651
|
+
key: `fleet:${config.fleet.namespace}`,
|
|
2652
|
+
apiKey: config.fleet.api_key,
|
|
2653
|
+
namespace: config.fleet.namespace,
|
|
2654
|
+
siteId: config.site_id,
|
|
2655
|
+
isFleet: true
|
|
2656
|
+
};
|
|
2657
|
+
}
|
|
2481
2658
|
return {
|
|
2482
|
-
|
|
2659
|
+
key: `default:${config.namespace}`,
|
|
2660
|
+
apiKey: config.candengo_api_key,
|
|
2483
2661
|
namespace: config.namespace,
|
|
2662
|
+
siteId: config.site_id,
|
|
2663
|
+
isFleet: false
|
|
2664
|
+
};
|
|
2665
|
+
}
|
|
2666
|
+
|
|
2667
|
+
// src/sync/push.ts
|
|
2668
|
+
function buildChatVectorDocument(chat, config, project, target = resolveSyncTarget(config, project.name)) {
|
|
2669
|
+
return {
|
|
2670
|
+
site_id: target.siteId,
|
|
2671
|
+
namespace: target.namespace,
|
|
2484
2672
|
source_type: "chat",
|
|
2485
2673
|
source_id: buildSourceId(config, chat.id, "chat"),
|
|
2486
2674
|
content: chat.content,
|
|
@@ -2501,7 +2689,7 @@ function buildChatVectorDocument(chat, config, project) {
|
|
|
2501
2689
|
}
|
|
2502
2690
|
};
|
|
2503
2691
|
}
|
|
2504
|
-
function buildVectorDocument(obs, config, project) {
|
|
2692
|
+
function buildVectorDocument(obs, config, project, target = resolveSyncTarget(config, project.name)) {
|
|
2505
2693
|
const parts = [obs.title];
|
|
2506
2694
|
if (obs.narrative)
|
|
2507
2695
|
parts.push(obs.narrative);
|
|
@@ -2518,8 +2706,8 @@ function buildVectorDocument(obs, config, project) {
|
|
|
2518
2706
|
}
|
|
2519
2707
|
}
|
|
2520
2708
|
return {
|
|
2521
|
-
site_id:
|
|
2522
|
-
namespace:
|
|
2709
|
+
site_id: target.siteId,
|
|
2710
|
+
namespace: target.namespace,
|
|
2523
2711
|
source_type: obs.type,
|
|
2524
2712
|
source_id: buildSourceId(config, obs.id),
|
|
2525
2713
|
content: parts.join(`
|
|
@@ -2550,7 +2738,10 @@ function buildVectorDocument(obs, config, project) {
|
|
|
2550
2738
|
}
|
|
2551
2739
|
};
|
|
2552
2740
|
}
|
|
2553
|
-
function buildSummaryVectorDocument(summary, config, project,
|
|
2741
|
+
function buildSummaryVectorDocument(summary, config, project, targetOrObservations = resolveSyncTarget(config, project.name), observationsOrCaptureContext = [], captureContext) {
|
|
2742
|
+
const target = Array.isArray(targetOrObservations) ? resolveSyncTarget(config, project.name) : targetOrObservations;
|
|
2743
|
+
const observations = Array.isArray(targetOrObservations) ? targetOrObservations : Array.isArray(observationsOrCaptureContext) ? observationsOrCaptureContext : [];
|
|
2744
|
+
const resolvedCaptureContext = Array.isArray(observationsOrCaptureContext) ? captureContext : observationsOrCaptureContext;
|
|
2554
2745
|
const parts = [];
|
|
2555
2746
|
if (summary.request)
|
|
2556
2747
|
parts.push(`Request: ${summary.request}`);
|
|
@@ -2563,9 +2754,17 @@ function buildSummaryVectorDocument(summary, config, project, observations = [],
|
|
|
2563
2754
|
if (summary.next_steps)
|
|
2564
2755
|
parts.push(`Next Steps: ${summary.next_steps}`);
|
|
2565
2756
|
const valueSignals = computeSessionValueSignals(observations, []);
|
|
2757
|
+
const observationSourceTools = resolvedCaptureContext?.observation_source_tools ?? summarizeObservationSourceTools(observations);
|
|
2758
|
+
const latestObservationPromptNumber = resolvedCaptureContext?.latest_observation_prompt_number ?? observations.reduce((latest, obs) => {
|
|
2759
|
+
if (typeof obs.source_prompt_number !== "number")
|
|
2760
|
+
return latest;
|
|
2761
|
+
if (latest === null || obs.source_prompt_number > latest)
|
|
2762
|
+
return obs.source_prompt_number;
|
|
2763
|
+
return latest;
|
|
2764
|
+
}, null);
|
|
2566
2765
|
return {
|
|
2567
|
-
site_id:
|
|
2568
|
-
namespace:
|
|
2766
|
+
site_id: target.siteId,
|
|
2767
|
+
namespace: target.namespace,
|
|
2569
2768
|
source_type: "summary",
|
|
2570
2769
|
source_id: buildSourceId(config, summary.id, "summary"),
|
|
2571
2770
|
content: parts.join(`
|
|
@@ -2587,18 +2786,18 @@ function buildSummaryVectorDocument(summary, config, project, observations = [],
|
|
|
2587
2786
|
learned_items: extractSectionItems(summary.learned),
|
|
2588
2787
|
completed_items: extractSectionItems(summary.completed),
|
|
2589
2788
|
next_step_items: extractSectionItems(summary.next_steps),
|
|
2590
|
-
prompt_count:
|
|
2591
|
-
tool_event_count:
|
|
2592
|
-
capture_state:
|
|
2593
|
-
recent_request_prompts:
|
|
2594
|
-
latest_request:
|
|
2595
|
-
current_thread:
|
|
2596
|
-
recent_tool_names:
|
|
2597
|
-
recent_tool_commands:
|
|
2598
|
-
hot_files:
|
|
2599
|
-
recent_outcomes:
|
|
2600
|
-
observation_source_tools:
|
|
2601
|
-
latest_observation_prompt_number:
|
|
2789
|
+
prompt_count: resolvedCaptureContext?.prompt_count ?? 0,
|
|
2790
|
+
tool_event_count: resolvedCaptureContext?.tool_event_count ?? 0,
|
|
2791
|
+
capture_state: resolvedCaptureContext?.capture_state ?? "summary-only",
|
|
2792
|
+
recent_request_prompts: resolvedCaptureContext?.recent_request_prompts ?? [],
|
|
2793
|
+
latest_request: resolvedCaptureContext?.latest_request ?? null,
|
|
2794
|
+
current_thread: resolvedCaptureContext?.current_thread ?? null,
|
|
2795
|
+
recent_tool_names: resolvedCaptureContext?.recent_tool_names ?? [],
|
|
2796
|
+
recent_tool_commands: resolvedCaptureContext?.recent_tool_commands ?? [],
|
|
2797
|
+
hot_files: resolvedCaptureContext?.hot_files ?? [],
|
|
2798
|
+
recent_outcomes: resolvedCaptureContext?.recent_outcomes ?? [],
|
|
2799
|
+
observation_source_tools: observationSourceTools,
|
|
2800
|
+
latest_observation_prompt_number: latestObservationPromptNumber,
|
|
2602
2801
|
decisions_count: valueSignals.decisions_count,
|
|
2603
2802
|
lessons_count: valueSignals.lessons_count,
|
|
2604
2803
|
discoveries_count: valueSignals.discoveries_count,
|
|
@@ -2612,7 +2811,7 @@ function buildSummaryVectorDocument(summary, config, project, observations = [],
|
|
|
2612
2811
|
}
|
|
2613
2812
|
};
|
|
2614
2813
|
}
|
|
2615
|
-
async function pushOutbox(db,
|
|
2814
|
+
async function pushOutbox(db, config, batchSize = 50) {
|
|
2616
2815
|
const entries = getPendingEntries(db, batchSize);
|
|
2617
2816
|
let pushed = 0;
|
|
2618
2817
|
let failed = 0;
|
|
@@ -2639,11 +2838,12 @@ async function pushOutbox(db, client, config, batchSize = 50) {
|
|
|
2639
2838
|
const summaryObservations = db.getObservationsBySession(summary.session_id);
|
|
2640
2839
|
const sessionPrompts = db.getSessionUserPrompts(summary.session_id, 20);
|
|
2641
2840
|
const sessionToolEvents = db.getSessionToolEvents(summary.session_id, 20);
|
|
2841
|
+
const target2 = resolveSyncTarget(config, project2.name);
|
|
2642
2842
|
const doc2 = buildSummaryVectorDocument(summary, config, {
|
|
2643
2843
|
canonical_id: project2.canonical_id,
|
|
2644
2844
|
name: project2.name
|
|
2645
|
-
}, summaryObservations, buildSessionHandoffMetadata(sessionPrompts, sessionToolEvents, summaryObservations));
|
|
2646
|
-
batch.push({ entryId: entry.id, doc: doc2 });
|
|
2845
|
+
}, target2, summaryObservations, buildSessionHandoffMetadata(sessionPrompts, sessionToolEvents, summaryObservations));
|
|
2846
|
+
batch.push({ entryId: entry.id, doc: maybeScrubFleetDocument(doc2, target2), target: target2 });
|
|
2647
2847
|
continue;
|
|
2648
2848
|
}
|
|
2649
2849
|
if (entry.record_type === "chat_message") {
|
|
@@ -2665,11 +2865,12 @@ async function pushOutbox(db, client, config, batchSize = 50) {
|
|
|
2665
2865
|
continue;
|
|
2666
2866
|
}
|
|
2667
2867
|
markSyncing(db, entry.id);
|
|
2868
|
+
const target2 = resolveSyncTarget(config, project2.name);
|
|
2668
2869
|
const doc2 = buildChatVectorDocument(chat, config, {
|
|
2669
2870
|
canonical_id: project2.canonical_id,
|
|
2670
2871
|
name: project2.name
|
|
2671
|
-
});
|
|
2672
|
-
batch.push({ entryId: entry.id, doc: doc2 });
|
|
2872
|
+
}, target2);
|
|
2873
|
+
batch.push({ entryId: entry.id, doc: maybeScrubFleetDocument(doc2, target2), target: target2 });
|
|
2673
2874
|
continue;
|
|
2674
2875
|
}
|
|
2675
2876
|
if (entry.record_type !== "observation") {
|
|
@@ -2699,30 +2900,33 @@ async function pushOutbox(db, client, config, batchSize = 50) {
|
|
|
2699
2900
|
continue;
|
|
2700
2901
|
}
|
|
2701
2902
|
markSyncing(db, entry.id);
|
|
2903
|
+
const target = resolveSyncTarget(config, project.name);
|
|
2702
2904
|
const doc = buildVectorDocument(obs, config, {
|
|
2703
2905
|
canonical_id: project.canonical_id,
|
|
2704
2906
|
name: project.name
|
|
2705
|
-
});
|
|
2706
|
-
batch.push({ entryId: entry.id, doc });
|
|
2907
|
+
}, target);
|
|
2908
|
+
batch.push({ entryId: entry.id, doc: maybeScrubFleetDocument(doc, target), target });
|
|
2707
2909
|
}
|
|
2708
2910
|
if (batch.length === 0)
|
|
2709
2911
|
return { pushed, failed, skipped };
|
|
2710
|
-
|
|
2711
|
-
|
|
2712
|
-
|
|
2713
|
-
|
|
2714
|
-
|
|
2715
|
-
|
|
2716
|
-
|
|
2717
|
-
}
|
|
2718
|
-
}
|
|
2719
|
-
markSynced(db, entryId);
|
|
2720
|
-
pushed++;
|
|
2912
|
+
const grouped = new Map;
|
|
2913
|
+
for (const item of batch) {
|
|
2914
|
+
const existing = grouped.get(item.target.key);
|
|
2915
|
+
if (existing) {
|
|
2916
|
+
existing.items.push(item);
|
|
2917
|
+
} else {
|
|
2918
|
+
grouped.set(item.target.key, { target: item.target, items: [item] });
|
|
2721
2919
|
}
|
|
2722
|
-
}
|
|
2723
|
-
|
|
2724
|
-
|
|
2725
|
-
|
|
2920
|
+
}
|
|
2921
|
+
for (const { target, items } of grouped.values()) {
|
|
2922
|
+
const client = new VectorClient(config, {
|
|
2923
|
+
apiKey: target.apiKey,
|
|
2924
|
+
namespace: target.namespace,
|
|
2925
|
+
siteId: target.siteId
|
|
2926
|
+
});
|
|
2927
|
+
try {
|
|
2928
|
+
await client.batchIngest(items.map((b) => b.doc));
|
|
2929
|
+
for (const { entryId, doc } of items) {
|
|
2726
2930
|
if (doc.source_type === "chat") {
|
|
2727
2931
|
const localId = typeof doc.metadata?.local_id === "number" ? doc.metadata.local_id : null;
|
|
2728
2932
|
if (localId !== null) {
|
|
@@ -2731,14 +2935,47 @@ async function pushOutbox(db, client, config, batchSize = 50) {
|
|
|
2731
2935
|
}
|
|
2732
2936
|
markSynced(db, entryId);
|
|
2733
2937
|
pushed++;
|
|
2734
|
-
}
|
|
2735
|
-
|
|
2736
|
-
|
|
2938
|
+
}
|
|
2939
|
+
} catch {
|
|
2940
|
+
for (const { entryId, doc } of items) {
|
|
2941
|
+
try {
|
|
2942
|
+
await client.ingest(doc);
|
|
2943
|
+
if (doc.source_type === "chat") {
|
|
2944
|
+
const localId = typeof doc.metadata?.local_id === "number" ? doc.metadata.local_id : null;
|
|
2945
|
+
if (localId !== null) {
|
|
2946
|
+
db.db.query("UPDATE chat_messages SET remote_source_id = ? WHERE id = ?").run(doc.source_id, localId);
|
|
2947
|
+
}
|
|
2948
|
+
}
|
|
2949
|
+
markSynced(db, entryId);
|
|
2950
|
+
pushed++;
|
|
2951
|
+
} catch (err) {
|
|
2952
|
+
markFailed(db, entryId, err instanceof Error ? err.message : String(err));
|
|
2953
|
+
failed++;
|
|
2954
|
+
}
|
|
2737
2955
|
}
|
|
2738
2956
|
}
|
|
2739
2957
|
}
|
|
2740
2958
|
return { pushed, failed, skipped };
|
|
2741
2959
|
}
|
|
2960
|
+
function maybeScrubFleetDocument(doc, target) {
|
|
2961
|
+
if (!target.isFleet)
|
|
2962
|
+
return doc;
|
|
2963
|
+
return {
|
|
2964
|
+
...doc,
|
|
2965
|
+
content: scrubFleetIdentifiers(doc.content),
|
|
2966
|
+
metadata: scrubFleetMetadata(doc.metadata)
|
|
2967
|
+
};
|
|
2968
|
+
}
|
|
2969
|
+
function scrubFleetMetadata(value) {
|
|
2970
|
+
if (typeof value === "string")
|
|
2971
|
+
return scrubFleetIdentifiers(value);
|
|
2972
|
+
if (Array.isArray(value))
|
|
2973
|
+
return value.map((item) => scrubFleetMetadata(item));
|
|
2974
|
+
if (value && typeof value === "object") {
|
|
2975
|
+
return Object.fromEntries(Object.entries(value).map(([key, item]) => [key, scrubFleetMetadata(item)]));
|
|
2976
|
+
}
|
|
2977
|
+
return value;
|
|
2978
|
+
}
|
|
2742
2979
|
function countPresentSections(summary) {
|
|
2743
2980
|
return [
|
|
2744
2981
|
summary.request,
|
|
@@ -2751,6 +2988,20 @@ function countPresentSections(summary) {
|
|
|
2751
2988
|
function extractSectionItems(section) {
|
|
2752
2989
|
return extractSummaryItems(section, 4);
|
|
2753
2990
|
}
|
|
2991
|
+
function summarizeObservationSourceTools(observations) {
|
|
2992
|
+
const counts = new Map;
|
|
2993
|
+
for (const obs of observations) {
|
|
2994
|
+
const tool = obs.source_tool;
|
|
2995
|
+
if (!tool)
|
|
2996
|
+
continue;
|
|
2997
|
+
counts.set(tool, (counts.get(tool) ?? 0) + 1);
|
|
2998
|
+
}
|
|
2999
|
+
return Array.from(counts.entries()).map(([tool, count]) => ({ tool, count })).sort((a, b) => {
|
|
3000
|
+
if (b.count !== a.count)
|
|
3001
|
+
return b.count - a.count;
|
|
3002
|
+
return a.tool.localeCompare(b.tool);
|
|
3003
|
+
});
|
|
3004
|
+
}
|
|
2754
3005
|
|
|
2755
3006
|
// src/embeddings/embedder.ts
|
|
2756
3007
|
var _available = null;
|
|
@@ -2862,7 +3113,7 @@ async function pushOnce(db, config) {
|
|
|
2862
3113
|
return 0;
|
|
2863
3114
|
try {
|
|
2864
3115
|
const client = new VectorClient(config);
|
|
2865
|
-
const result = await pushOutbox(db,
|
|
3116
|
+
const result = await pushOutbox(db, config, config.sync.batch_size);
|
|
2866
3117
|
await pullSettings(client, config);
|
|
2867
3118
|
return result.pushed;
|
|
2868
3119
|
} catch {
|
|
@@ -3087,7 +3338,7 @@ function buildBeacon(db, config, sessionId, metrics) {
|
|
|
3087
3338
|
sentinel_used: valueSignals.security_findings_count > 0,
|
|
3088
3339
|
risk_score: riskScore,
|
|
3089
3340
|
stacks_detected: stacks,
|
|
3090
|
-
client_version: "0.4.
|
|
3341
|
+
client_version: "0.4.39",
|
|
3091
3342
|
context_observations_injected: metrics?.contextObsInjected ?? 0,
|
|
3092
3343
|
context_total_available: metrics?.contextTotalAvailable ?? 0,
|
|
3093
3344
|
recall_attempts: metrics?.recallAttempts ?? 0,
|
|
@@ -3283,139 +3534,6 @@ import { homedir as homedir3 } from "node:os";
|
|
|
3283
3534
|
// src/tools/save.ts
|
|
3284
3535
|
import { relative, isAbsolute } from "node:path";
|
|
3285
3536
|
|
|
3286
|
-
// src/capture/scrubber.ts
|
|
3287
|
-
var DEFAULT_PATTERNS = [
|
|
3288
|
-
{
|
|
3289
|
-
source: "sk-[a-zA-Z0-9]{20,}",
|
|
3290
|
-
flags: "g",
|
|
3291
|
-
replacement: "[REDACTED_API_KEY]",
|
|
3292
|
-
description: "OpenAI API keys",
|
|
3293
|
-
category: "api_key",
|
|
3294
|
-
severity: "critical"
|
|
3295
|
-
},
|
|
3296
|
-
{
|
|
3297
|
-
source: "Bearer [a-zA-Z0-9\\-._~+/]+=*",
|
|
3298
|
-
flags: "g",
|
|
3299
|
-
replacement: "[REDACTED_BEARER]",
|
|
3300
|
-
description: "Bearer auth tokens",
|
|
3301
|
-
category: "token",
|
|
3302
|
-
severity: "medium"
|
|
3303
|
-
},
|
|
3304
|
-
{
|
|
3305
|
-
source: "password[=:]\\s*\\S+",
|
|
3306
|
-
flags: "gi",
|
|
3307
|
-
replacement: "password=[REDACTED]",
|
|
3308
|
-
description: "Passwords in config",
|
|
3309
|
-
category: "password",
|
|
3310
|
-
severity: "high"
|
|
3311
|
-
},
|
|
3312
|
-
{
|
|
3313
|
-
source: "postgresql://[^\\s]+",
|
|
3314
|
-
flags: "g",
|
|
3315
|
-
replacement: "[REDACTED_DB_URL]",
|
|
3316
|
-
description: "PostgreSQL connection strings",
|
|
3317
|
-
category: "db_url",
|
|
3318
|
-
severity: "high"
|
|
3319
|
-
},
|
|
3320
|
-
{
|
|
3321
|
-
source: "mongodb://[^\\s]+",
|
|
3322
|
-
flags: "g",
|
|
3323
|
-
replacement: "[REDACTED_DB_URL]",
|
|
3324
|
-
description: "MongoDB connection strings",
|
|
3325
|
-
category: "db_url",
|
|
3326
|
-
severity: "high"
|
|
3327
|
-
},
|
|
3328
|
-
{
|
|
3329
|
-
source: "mysql://[^\\s]+",
|
|
3330
|
-
flags: "g",
|
|
3331
|
-
replacement: "[REDACTED_DB_URL]",
|
|
3332
|
-
description: "MySQL connection strings",
|
|
3333
|
-
category: "db_url",
|
|
3334
|
-
severity: "high"
|
|
3335
|
-
},
|
|
3336
|
-
{
|
|
3337
|
-
source: "AKIA[A-Z0-9]{16}",
|
|
3338
|
-
flags: "g",
|
|
3339
|
-
replacement: "[REDACTED_AWS_KEY]",
|
|
3340
|
-
description: "AWS access keys",
|
|
3341
|
-
category: "api_key",
|
|
3342
|
-
severity: "critical"
|
|
3343
|
-
},
|
|
3344
|
-
{
|
|
3345
|
-
source: "ghp_[a-zA-Z0-9]{36}",
|
|
3346
|
-
flags: "g",
|
|
3347
|
-
replacement: "[REDACTED_GH_TOKEN]",
|
|
3348
|
-
description: "GitHub personal access tokens",
|
|
3349
|
-
category: "token",
|
|
3350
|
-
severity: "high"
|
|
3351
|
-
},
|
|
3352
|
-
{
|
|
3353
|
-
source: "gho_[a-zA-Z0-9]{36}",
|
|
3354
|
-
flags: "g",
|
|
3355
|
-
replacement: "[REDACTED_GH_TOKEN]",
|
|
3356
|
-
description: "GitHub OAuth tokens",
|
|
3357
|
-
category: "token",
|
|
3358
|
-
severity: "high"
|
|
3359
|
-
},
|
|
3360
|
-
{
|
|
3361
|
-
source: "github_pat_[a-zA-Z0-9_]{22,}",
|
|
3362
|
-
flags: "g",
|
|
3363
|
-
replacement: "[REDACTED_GH_TOKEN]",
|
|
3364
|
-
description: "GitHub fine-grained PATs",
|
|
3365
|
-
category: "token",
|
|
3366
|
-
severity: "high"
|
|
3367
|
-
},
|
|
3368
|
-
{
|
|
3369
|
-
source: "cvk_[a-f0-9]{64}",
|
|
3370
|
-
flags: "g",
|
|
3371
|
-
replacement: "[REDACTED_CANDENGO_KEY]",
|
|
3372
|
-
description: "Candengo API keys",
|
|
3373
|
-
category: "api_key",
|
|
3374
|
-
severity: "critical"
|
|
3375
|
-
},
|
|
3376
|
-
{
|
|
3377
|
-
source: "xox[bpras]-[a-zA-Z0-9\\-]+",
|
|
3378
|
-
flags: "g",
|
|
3379
|
-
replacement: "[REDACTED_SLACK_TOKEN]",
|
|
3380
|
-
description: "Slack tokens",
|
|
3381
|
-
category: "token",
|
|
3382
|
-
severity: "high"
|
|
3383
|
-
}
|
|
3384
|
-
];
|
|
3385
|
-
function compileCustomPatterns(patterns) {
|
|
3386
|
-
const compiled = [];
|
|
3387
|
-
for (const pattern of patterns) {
|
|
3388
|
-
try {
|
|
3389
|
-
new RegExp(pattern);
|
|
3390
|
-
compiled.push({
|
|
3391
|
-
source: pattern,
|
|
3392
|
-
flags: "g",
|
|
3393
|
-
replacement: "[REDACTED_CUSTOM]",
|
|
3394
|
-
description: `Custom pattern: ${pattern}`,
|
|
3395
|
-
category: "custom",
|
|
3396
|
-
severity: "medium"
|
|
3397
|
-
});
|
|
3398
|
-
} catch {}
|
|
3399
|
-
}
|
|
3400
|
-
return compiled;
|
|
3401
|
-
}
|
|
3402
|
-
function scrubSecrets(text, customPatterns = []) {
|
|
3403
|
-
let result = text;
|
|
3404
|
-
const allPatterns = [...DEFAULT_PATTERNS, ...compileCustomPatterns(customPatterns)];
|
|
3405
|
-
for (const pattern of allPatterns) {
|
|
3406
|
-
result = result.replace(new RegExp(pattern.source, pattern.flags), pattern.replacement);
|
|
3407
|
-
}
|
|
3408
|
-
return result;
|
|
3409
|
-
}
|
|
3410
|
-
function containsSecrets(text, customPatterns = []) {
|
|
3411
|
-
const allPatterns = [...DEFAULT_PATTERNS, ...compileCustomPatterns(customPatterns)];
|
|
3412
|
-
for (const pattern of allPatterns) {
|
|
3413
|
-
if (new RegExp(pattern.source, pattern.flags).test(text))
|
|
3414
|
-
return true;
|
|
3415
|
-
}
|
|
3416
|
-
return false;
|
|
3417
|
-
}
|
|
3418
|
-
|
|
3419
3537
|
// src/capture/quality.ts
|
|
3420
3538
|
var QUALITY_THRESHOLD = 0.1;
|
|
3421
3539
|
function scoreQuality(input) {
|
|
@@ -3833,7 +3951,8 @@ async function saveObservation(db, config, input) {
|
|
|
3833
3951
|
const factsJson = structuredFacts.length > 0 ? config.scrubbing.enabled ? scrubSecrets(JSON.stringify(structuredFacts), customPatterns) : JSON.stringify(structuredFacts) : null;
|
|
3834
3952
|
const filesReadJson = filesRead ? JSON.stringify(filesRead) : null;
|
|
3835
3953
|
const filesModifiedJson = filesModified ? JSON.stringify(filesModified) : null;
|
|
3836
|
-
|
|
3954
|
+
const fleetProject = isFleetProjectName(project.name, config);
|
|
3955
|
+
let sensitivity = input.sensitivity ?? (fleetProject ? "shared" : config.scrubbing.default_sensitivity);
|
|
3837
3956
|
if (config.scrubbing.enabled && containsSecrets([input.title, input.narrative, JSON.stringify(input.facts)].filter(Boolean).join(" "), customPatterns)) {
|
|
3838
3957
|
if (sensitivity === "shared") {
|
|
3839
3958
|
sensitivity = "personal";
|