engrm 0.4.38 → 0.4.39

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -389,6 +389,16 @@ function createDefaultConfig() {
389
389
  },
390
390
  transcript_analysis: {
391
391
  enabled: false
392
+ },
393
+ http: {
394
+ enabled: false,
395
+ port: 3767,
396
+ bearer_tokens: []
397
+ },
398
+ fleet: {
399
+ project_name: "shared-experience",
400
+ namespace: "",
401
+ api_key: ""
392
402
  }
393
403
  };
394
404
  }
@@ -450,6 +460,16 @@ function loadConfig() {
450
460
  },
451
461
  transcript_analysis: {
452
462
  enabled: asBool(config["transcript_analysis"]?.["enabled"], defaults.transcript_analysis.enabled)
463
+ },
464
+ http: {
465
+ enabled: asBool(config["http"]?.["enabled"], defaults.http.enabled),
466
+ port: asNumber(config["http"]?.["port"], defaults.http.port),
467
+ bearer_tokens: asStringArray(config["http"]?.["bearer_tokens"], defaults.http.bearer_tokens)
468
+ },
469
+ fleet: {
470
+ project_name: asString(config["fleet"]?.["project_name"], defaults.fleet.project_name),
471
+ namespace: asString(config["fleet"]?.["namespace"], defaults.fleet.namespace),
472
+ api_key: asString(config["fleet"]?.["api_key"], defaults.fleet.api_key)
453
473
  }
454
474
  };
455
475
  }
@@ -2251,16 +2271,16 @@ class VectorClient {
2251
2271
  apiKey;
2252
2272
  siteId;
2253
2273
  namespace;
2254
- constructor(config) {
2274
+ constructor(config, overrides = {}) {
2255
2275
  const baseUrl = getBaseUrl(config);
2256
- const apiKey = getApiKey(config);
2276
+ const apiKey = overrides.apiKey ?? getApiKey(config);
2257
2277
  if (!baseUrl || !apiKey) {
2258
2278
  throw new Error("VectorClient requires candengo_url and candengo_api_key");
2259
2279
  }
2260
2280
  this.baseUrl = baseUrl.replace(/\/$/, "");
2261
2281
  this.apiKey = apiKey;
2262
- this.siteId = config.site_id;
2263
- this.namespace = config.namespace;
2282
+ this.siteId = overrides.siteId ?? config.site_id;
2283
+ this.namespace = overrides.namespace ?? config.namespace;
2264
2284
  }
2265
2285
  static isConfigured(config) {
2266
2286
  return getApiKey(config) !== null && getBaseUrl(config) !== null;
@@ -2476,11 +2496,179 @@ function parseJsonArray2(value) {
2476
2496
  }
2477
2497
  }
2478
2498
 
2479
- // src/sync/push.ts
2480
- function buildChatVectorDocument(chat, config, project) {
2499
+ // src/capture/scrubber.ts
2500
+ var DEFAULT_PATTERNS = [
2501
+ {
2502
+ source: "sk-[a-zA-Z0-9]{20,}",
2503
+ flags: "g",
2504
+ replacement: "[REDACTED_API_KEY]",
2505
+ description: "OpenAI API keys",
2506
+ category: "api_key",
2507
+ severity: "critical"
2508
+ },
2509
+ {
2510
+ source: "Bearer [a-zA-Z0-9\\-._~+/]+=*",
2511
+ flags: "g",
2512
+ replacement: "[REDACTED_BEARER]",
2513
+ description: "Bearer auth tokens",
2514
+ category: "token",
2515
+ severity: "medium"
2516
+ },
2517
+ {
2518
+ source: "password[=:]\\s*\\S+",
2519
+ flags: "gi",
2520
+ replacement: "password=[REDACTED]",
2521
+ description: "Passwords in config",
2522
+ category: "password",
2523
+ severity: "high"
2524
+ },
2525
+ {
2526
+ source: "postgresql://[^\\s]+",
2527
+ flags: "g",
2528
+ replacement: "[REDACTED_DB_URL]",
2529
+ description: "PostgreSQL connection strings",
2530
+ category: "db_url",
2531
+ severity: "high"
2532
+ },
2533
+ {
2534
+ source: "mongodb://[^\\s]+",
2535
+ flags: "g",
2536
+ replacement: "[REDACTED_DB_URL]",
2537
+ description: "MongoDB connection strings",
2538
+ category: "db_url",
2539
+ severity: "high"
2540
+ },
2541
+ {
2542
+ source: "mysql://[^\\s]+",
2543
+ flags: "g",
2544
+ replacement: "[REDACTED_DB_URL]",
2545
+ description: "MySQL connection strings",
2546
+ category: "db_url",
2547
+ severity: "high"
2548
+ },
2549
+ {
2550
+ source: "AKIA[A-Z0-9]{16}",
2551
+ flags: "g",
2552
+ replacement: "[REDACTED_AWS_KEY]",
2553
+ description: "AWS access keys",
2554
+ category: "api_key",
2555
+ severity: "critical"
2556
+ },
2557
+ {
2558
+ source: "ghp_[a-zA-Z0-9]{36}",
2559
+ flags: "g",
2560
+ replacement: "[REDACTED_GH_TOKEN]",
2561
+ description: "GitHub personal access tokens",
2562
+ category: "token",
2563
+ severity: "high"
2564
+ },
2565
+ {
2566
+ source: "gho_[a-zA-Z0-9]{36}",
2567
+ flags: "g",
2568
+ replacement: "[REDACTED_GH_TOKEN]",
2569
+ description: "GitHub OAuth tokens",
2570
+ category: "token",
2571
+ severity: "high"
2572
+ },
2573
+ {
2574
+ source: "github_pat_[a-zA-Z0-9_]{22,}",
2575
+ flags: "g",
2576
+ replacement: "[REDACTED_GH_TOKEN]",
2577
+ description: "GitHub fine-grained PATs",
2578
+ category: "token",
2579
+ severity: "high"
2580
+ },
2581
+ {
2582
+ source: "cvk_[a-f0-9]{64}",
2583
+ flags: "g",
2584
+ replacement: "[REDACTED_CANDENGO_KEY]",
2585
+ description: "Candengo API keys",
2586
+ category: "api_key",
2587
+ severity: "critical"
2588
+ },
2589
+ {
2590
+ source: "xox[bpras]-[a-zA-Z0-9\\-]+",
2591
+ flags: "g",
2592
+ replacement: "[REDACTED_SLACK_TOKEN]",
2593
+ description: "Slack tokens",
2594
+ category: "token",
2595
+ severity: "high"
2596
+ }
2597
+ ];
2598
+ function compileCustomPatterns(patterns) {
2599
+ const compiled = [];
2600
+ for (const pattern of patterns) {
2601
+ try {
2602
+ new RegExp(pattern);
2603
+ compiled.push({
2604
+ source: pattern,
2605
+ flags: "g",
2606
+ replacement: "[REDACTED_CUSTOM]",
2607
+ description: `Custom pattern: ${pattern}`,
2608
+ category: "custom",
2609
+ severity: "medium"
2610
+ });
2611
+ } catch {}
2612
+ }
2613
+ return compiled;
2614
+ }
2615
+ function scrubSecrets(text, customPatterns = []) {
2616
+ let result = text;
2617
+ const allPatterns = [...DEFAULT_PATTERNS, ...compileCustomPatterns(customPatterns)];
2618
+ for (const pattern of allPatterns) {
2619
+ result = result.replace(new RegExp(pattern.source, pattern.flags), pattern.replacement);
2620
+ }
2621
+ return result;
2622
+ }
2623
+ function containsSecrets(text, customPatterns = []) {
2624
+ const allPatterns = [...DEFAULT_PATTERNS, ...compileCustomPatterns(customPatterns)];
2625
+ for (const pattern of allPatterns) {
2626
+ if (new RegExp(pattern.source, pattern.flags).test(text))
2627
+ return true;
2628
+ }
2629
+ return false;
2630
+ }
2631
+ var FLEET_HOSTNAME_PATTERN = /\b(?=.{1,253}\b)(?!-)(?:[a-z0-9](?:[a-z0-9-]{0,61}[a-z0-9])?\.)+[a-z]{2,63}\b/gi;
2632
+ var FLEET_IP_PATTERN = /\b(?:\d{1,3}\.){3}\d{1,3}\b/g;
2633
+ var FLEET_MAC_PATTERN = /\b(?:[0-9a-f]{2}[:-]){5}[0-9a-f]{2}\b/gi;
2634
+ function scrubFleetIdentifiers(text) {
2635
+ return text.replace(FLEET_MAC_PATTERN, "[REDACTED_MAC]").replace(FLEET_IP_PATTERN, "[REDACTED_IP]").replace(FLEET_HOSTNAME_PATTERN, "[REDACTED_HOSTNAME]");
2636
+ }
2637
+
2638
+ // src/sync/targets.ts
2639
+ function isFleetProjectName(projectName, config) {
2640
+ const fleetProjectName = config.fleet?.project_name ?? "shared-experience";
2641
+ if (!projectName || !fleetProjectName)
2642
+ return false;
2643
+ return projectName.trim().toLowerCase() === fleetProjectName.trim().toLowerCase();
2644
+ }
2645
+ function hasFleetTarget(config) {
2646
+ return Boolean(config.fleet?.namespace?.trim() && config.fleet?.api_key?.trim() && (config.fleet?.project_name ?? "shared-experience").trim());
2647
+ }
2648
+ function resolveSyncTarget(config, projectName) {
2649
+ if (isFleetProjectName(projectName, config) && hasFleetTarget(config)) {
2650
+ return {
2651
+ key: `fleet:${config.fleet.namespace}`,
2652
+ apiKey: config.fleet.api_key,
2653
+ namespace: config.fleet.namespace,
2654
+ siteId: config.site_id,
2655
+ isFleet: true
2656
+ };
2657
+ }
2481
2658
  return {
2482
- site_id: config.site_id,
2659
+ key: `default:${config.namespace}`,
2660
+ apiKey: config.candengo_api_key,
2483
2661
  namespace: config.namespace,
2662
+ siteId: config.site_id,
2663
+ isFleet: false
2664
+ };
2665
+ }
2666
+
2667
+ // src/sync/push.ts
2668
+ function buildChatVectorDocument(chat, config, project, target = resolveSyncTarget(config, project.name)) {
2669
+ return {
2670
+ site_id: target.siteId,
2671
+ namespace: target.namespace,
2484
2672
  source_type: "chat",
2485
2673
  source_id: buildSourceId(config, chat.id, "chat"),
2486
2674
  content: chat.content,
@@ -2501,7 +2689,7 @@ function buildChatVectorDocument(chat, config, project) {
2501
2689
  }
2502
2690
  };
2503
2691
  }
2504
- function buildVectorDocument(obs, config, project) {
2692
+ function buildVectorDocument(obs, config, project, target = resolveSyncTarget(config, project.name)) {
2505
2693
  const parts = [obs.title];
2506
2694
  if (obs.narrative)
2507
2695
  parts.push(obs.narrative);
@@ -2518,8 +2706,8 @@ function buildVectorDocument(obs, config, project) {
2518
2706
  }
2519
2707
  }
2520
2708
  return {
2521
- site_id: config.site_id,
2522
- namespace: config.namespace,
2709
+ site_id: target.siteId,
2710
+ namespace: target.namespace,
2523
2711
  source_type: obs.type,
2524
2712
  source_id: buildSourceId(config, obs.id),
2525
2713
  content: parts.join(`
@@ -2550,7 +2738,10 @@ function buildVectorDocument(obs, config, project) {
2550
2738
  }
2551
2739
  };
2552
2740
  }
2553
- function buildSummaryVectorDocument(summary, config, project, observations = [], captureContext) {
2741
+ function buildSummaryVectorDocument(summary, config, project, targetOrObservations = resolveSyncTarget(config, project.name), observationsOrCaptureContext = [], captureContext) {
2742
+ const target = Array.isArray(targetOrObservations) ? resolveSyncTarget(config, project.name) : targetOrObservations;
2743
+ const observations = Array.isArray(targetOrObservations) ? targetOrObservations : Array.isArray(observationsOrCaptureContext) ? observationsOrCaptureContext : [];
2744
+ const resolvedCaptureContext = Array.isArray(observationsOrCaptureContext) ? captureContext : observationsOrCaptureContext;
2554
2745
  const parts = [];
2555
2746
  if (summary.request)
2556
2747
  parts.push(`Request: ${summary.request}`);
@@ -2563,9 +2754,17 @@ function buildSummaryVectorDocument(summary, config, project, observations = [],
2563
2754
  if (summary.next_steps)
2564
2755
  parts.push(`Next Steps: ${summary.next_steps}`);
2565
2756
  const valueSignals = computeSessionValueSignals(observations, []);
2757
+ const observationSourceTools = resolvedCaptureContext?.observation_source_tools ?? summarizeObservationSourceTools(observations);
2758
+ const latestObservationPromptNumber = resolvedCaptureContext?.latest_observation_prompt_number ?? observations.reduce((latest, obs) => {
2759
+ if (typeof obs.source_prompt_number !== "number")
2760
+ return latest;
2761
+ if (latest === null || obs.source_prompt_number > latest)
2762
+ return obs.source_prompt_number;
2763
+ return latest;
2764
+ }, null);
2566
2765
  return {
2567
- site_id: config.site_id,
2568
- namespace: config.namespace,
2766
+ site_id: target.siteId,
2767
+ namespace: target.namespace,
2569
2768
  source_type: "summary",
2570
2769
  source_id: buildSourceId(config, summary.id, "summary"),
2571
2770
  content: parts.join(`
@@ -2587,18 +2786,18 @@ function buildSummaryVectorDocument(summary, config, project, observations = [],
2587
2786
  learned_items: extractSectionItems(summary.learned),
2588
2787
  completed_items: extractSectionItems(summary.completed),
2589
2788
  next_step_items: extractSectionItems(summary.next_steps),
2590
- prompt_count: captureContext?.prompt_count ?? 0,
2591
- tool_event_count: captureContext?.tool_event_count ?? 0,
2592
- capture_state: captureContext?.capture_state ?? "summary-only",
2593
- recent_request_prompts: captureContext?.recent_request_prompts ?? [],
2594
- latest_request: captureContext?.latest_request ?? null,
2595
- current_thread: captureContext?.current_thread ?? null,
2596
- recent_tool_names: captureContext?.recent_tool_names ?? [],
2597
- recent_tool_commands: captureContext?.recent_tool_commands ?? [],
2598
- hot_files: captureContext?.hot_files ?? [],
2599
- recent_outcomes: captureContext?.recent_outcomes ?? [],
2600
- observation_source_tools: captureContext?.observation_source_tools ?? [],
2601
- latest_observation_prompt_number: captureContext?.latest_observation_prompt_number ?? null,
2789
+ prompt_count: resolvedCaptureContext?.prompt_count ?? 0,
2790
+ tool_event_count: resolvedCaptureContext?.tool_event_count ?? 0,
2791
+ capture_state: resolvedCaptureContext?.capture_state ?? "summary-only",
2792
+ recent_request_prompts: resolvedCaptureContext?.recent_request_prompts ?? [],
2793
+ latest_request: resolvedCaptureContext?.latest_request ?? null,
2794
+ current_thread: resolvedCaptureContext?.current_thread ?? null,
2795
+ recent_tool_names: resolvedCaptureContext?.recent_tool_names ?? [],
2796
+ recent_tool_commands: resolvedCaptureContext?.recent_tool_commands ?? [],
2797
+ hot_files: resolvedCaptureContext?.hot_files ?? [],
2798
+ recent_outcomes: resolvedCaptureContext?.recent_outcomes ?? [],
2799
+ observation_source_tools: observationSourceTools,
2800
+ latest_observation_prompt_number: latestObservationPromptNumber,
2602
2801
  decisions_count: valueSignals.decisions_count,
2603
2802
  lessons_count: valueSignals.lessons_count,
2604
2803
  discoveries_count: valueSignals.discoveries_count,
@@ -2612,7 +2811,7 @@ function buildSummaryVectorDocument(summary, config, project, observations = [],
2612
2811
  }
2613
2812
  };
2614
2813
  }
2615
- async function pushOutbox(db, client, config, batchSize = 50) {
2814
+ async function pushOutbox(db, config, batchSize = 50) {
2616
2815
  const entries = getPendingEntries(db, batchSize);
2617
2816
  let pushed = 0;
2618
2817
  let failed = 0;
@@ -2639,11 +2838,12 @@ async function pushOutbox(db, client, config, batchSize = 50) {
2639
2838
  const summaryObservations = db.getObservationsBySession(summary.session_id);
2640
2839
  const sessionPrompts = db.getSessionUserPrompts(summary.session_id, 20);
2641
2840
  const sessionToolEvents = db.getSessionToolEvents(summary.session_id, 20);
2841
+ const target2 = resolveSyncTarget(config, project2.name);
2642
2842
  const doc2 = buildSummaryVectorDocument(summary, config, {
2643
2843
  canonical_id: project2.canonical_id,
2644
2844
  name: project2.name
2645
- }, summaryObservations, buildSessionHandoffMetadata(sessionPrompts, sessionToolEvents, summaryObservations));
2646
- batch.push({ entryId: entry.id, doc: doc2 });
2845
+ }, target2, summaryObservations, buildSessionHandoffMetadata(sessionPrompts, sessionToolEvents, summaryObservations));
2846
+ batch.push({ entryId: entry.id, doc: maybeScrubFleetDocument(doc2, target2), target: target2 });
2647
2847
  continue;
2648
2848
  }
2649
2849
  if (entry.record_type === "chat_message") {
@@ -2665,11 +2865,12 @@ async function pushOutbox(db, client, config, batchSize = 50) {
2665
2865
  continue;
2666
2866
  }
2667
2867
  markSyncing(db, entry.id);
2868
+ const target2 = resolveSyncTarget(config, project2.name);
2668
2869
  const doc2 = buildChatVectorDocument(chat, config, {
2669
2870
  canonical_id: project2.canonical_id,
2670
2871
  name: project2.name
2671
- });
2672
- batch.push({ entryId: entry.id, doc: doc2 });
2872
+ }, target2);
2873
+ batch.push({ entryId: entry.id, doc: maybeScrubFleetDocument(doc2, target2), target: target2 });
2673
2874
  continue;
2674
2875
  }
2675
2876
  if (entry.record_type !== "observation") {
@@ -2699,30 +2900,33 @@ async function pushOutbox(db, client, config, batchSize = 50) {
2699
2900
  continue;
2700
2901
  }
2701
2902
  markSyncing(db, entry.id);
2903
+ const target = resolveSyncTarget(config, project.name);
2702
2904
  const doc = buildVectorDocument(obs, config, {
2703
2905
  canonical_id: project.canonical_id,
2704
2906
  name: project.name
2705
- });
2706
- batch.push({ entryId: entry.id, doc });
2907
+ }, target);
2908
+ batch.push({ entryId: entry.id, doc: maybeScrubFleetDocument(doc, target), target });
2707
2909
  }
2708
2910
  if (batch.length === 0)
2709
2911
  return { pushed, failed, skipped };
2710
- try {
2711
- await client.batchIngest(batch.map((b) => b.doc));
2712
- for (const { entryId, doc } of batch) {
2713
- if (doc.source_type === "chat") {
2714
- const localId = typeof doc.metadata?.local_id === "number" ? doc.metadata.local_id : null;
2715
- if (localId !== null) {
2716
- db.db.query("UPDATE chat_messages SET remote_source_id = ? WHERE id = ?").run(doc.source_id, localId);
2717
- }
2718
- }
2719
- markSynced(db, entryId);
2720
- pushed++;
2912
+ const grouped = new Map;
2913
+ for (const item of batch) {
2914
+ const existing = grouped.get(item.target.key);
2915
+ if (existing) {
2916
+ existing.items.push(item);
2917
+ } else {
2918
+ grouped.set(item.target.key, { target: item.target, items: [item] });
2721
2919
  }
2722
- } catch {
2723
- for (const { entryId, doc } of batch) {
2724
- try {
2725
- await client.ingest(doc);
2920
+ }
2921
+ for (const { target, items } of grouped.values()) {
2922
+ const client = new VectorClient(config, {
2923
+ apiKey: target.apiKey,
2924
+ namespace: target.namespace,
2925
+ siteId: target.siteId
2926
+ });
2927
+ try {
2928
+ await client.batchIngest(items.map((b) => b.doc));
2929
+ for (const { entryId, doc } of items) {
2726
2930
  if (doc.source_type === "chat") {
2727
2931
  const localId = typeof doc.metadata?.local_id === "number" ? doc.metadata.local_id : null;
2728
2932
  if (localId !== null) {
@@ -2731,14 +2935,47 @@ async function pushOutbox(db, client, config, batchSize = 50) {
2731
2935
  }
2732
2936
  markSynced(db, entryId);
2733
2937
  pushed++;
2734
- } catch (err) {
2735
- markFailed(db, entryId, err instanceof Error ? err.message : String(err));
2736
- failed++;
2938
+ }
2939
+ } catch {
2940
+ for (const { entryId, doc } of items) {
2941
+ try {
2942
+ await client.ingest(doc);
2943
+ if (doc.source_type === "chat") {
2944
+ const localId = typeof doc.metadata?.local_id === "number" ? doc.metadata.local_id : null;
2945
+ if (localId !== null) {
2946
+ db.db.query("UPDATE chat_messages SET remote_source_id = ? WHERE id = ?").run(doc.source_id, localId);
2947
+ }
2948
+ }
2949
+ markSynced(db, entryId);
2950
+ pushed++;
2951
+ } catch (err) {
2952
+ markFailed(db, entryId, err instanceof Error ? err.message : String(err));
2953
+ failed++;
2954
+ }
2737
2955
  }
2738
2956
  }
2739
2957
  }
2740
2958
  return { pushed, failed, skipped };
2741
2959
  }
2960
+ function maybeScrubFleetDocument(doc, target) {
2961
+ if (!target.isFleet)
2962
+ return doc;
2963
+ return {
2964
+ ...doc,
2965
+ content: scrubFleetIdentifiers(doc.content),
2966
+ metadata: scrubFleetMetadata(doc.metadata)
2967
+ };
2968
+ }
2969
+ function scrubFleetMetadata(value) {
2970
+ if (typeof value === "string")
2971
+ return scrubFleetIdentifiers(value);
2972
+ if (Array.isArray(value))
2973
+ return value.map((item) => scrubFleetMetadata(item));
2974
+ if (value && typeof value === "object") {
2975
+ return Object.fromEntries(Object.entries(value).map(([key, item]) => [key, scrubFleetMetadata(item)]));
2976
+ }
2977
+ return value;
2978
+ }
2742
2979
  function countPresentSections(summary) {
2743
2980
  return [
2744
2981
  summary.request,
@@ -2751,6 +2988,20 @@ function countPresentSections(summary) {
2751
2988
  function extractSectionItems(section) {
2752
2989
  return extractSummaryItems(section, 4);
2753
2990
  }
2991
+ function summarizeObservationSourceTools(observations) {
2992
+ const counts = new Map;
2993
+ for (const obs of observations) {
2994
+ const tool = obs.source_tool;
2995
+ if (!tool)
2996
+ continue;
2997
+ counts.set(tool, (counts.get(tool) ?? 0) + 1);
2998
+ }
2999
+ return Array.from(counts.entries()).map(([tool, count]) => ({ tool, count })).sort((a, b) => {
3000
+ if (b.count !== a.count)
3001
+ return b.count - a.count;
3002
+ return a.tool.localeCompare(b.tool);
3003
+ });
3004
+ }
2754
3005
 
2755
3006
  // src/embeddings/embedder.ts
2756
3007
  var _available = null;
@@ -2862,7 +3113,7 @@ async function pushOnce(db, config) {
2862
3113
  return 0;
2863
3114
  try {
2864
3115
  const client = new VectorClient(config);
2865
- const result = await pushOutbox(db, client, config, config.sync.batch_size);
3116
+ const result = await pushOutbox(db, config, config.sync.batch_size);
2866
3117
  await pullSettings(client, config);
2867
3118
  return result.pushed;
2868
3119
  } catch {
@@ -3087,7 +3338,7 @@ function buildBeacon(db, config, sessionId, metrics) {
3087
3338
  sentinel_used: valueSignals.security_findings_count > 0,
3088
3339
  risk_score: riskScore,
3089
3340
  stacks_detected: stacks,
3090
- client_version: "0.4.38",
3341
+ client_version: "0.4.39",
3091
3342
  context_observations_injected: metrics?.contextObsInjected ?? 0,
3092
3343
  context_total_available: metrics?.contextTotalAvailable ?? 0,
3093
3344
  recall_attempts: metrics?.recallAttempts ?? 0,
@@ -3283,139 +3534,6 @@ import { homedir as homedir3 } from "node:os";
3283
3534
  // src/tools/save.ts
3284
3535
  import { relative, isAbsolute } from "node:path";
3285
3536
 
3286
- // src/capture/scrubber.ts
3287
- var DEFAULT_PATTERNS = [
3288
- {
3289
- source: "sk-[a-zA-Z0-9]{20,}",
3290
- flags: "g",
3291
- replacement: "[REDACTED_API_KEY]",
3292
- description: "OpenAI API keys",
3293
- category: "api_key",
3294
- severity: "critical"
3295
- },
3296
- {
3297
- source: "Bearer [a-zA-Z0-9\\-._~+/]+=*",
3298
- flags: "g",
3299
- replacement: "[REDACTED_BEARER]",
3300
- description: "Bearer auth tokens",
3301
- category: "token",
3302
- severity: "medium"
3303
- },
3304
- {
3305
- source: "password[=:]\\s*\\S+",
3306
- flags: "gi",
3307
- replacement: "password=[REDACTED]",
3308
- description: "Passwords in config",
3309
- category: "password",
3310
- severity: "high"
3311
- },
3312
- {
3313
- source: "postgresql://[^\\s]+",
3314
- flags: "g",
3315
- replacement: "[REDACTED_DB_URL]",
3316
- description: "PostgreSQL connection strings",
3317
- category: "db_url",
3318
- severity: "high"
3319
- },
3320
- {
3321
- source: "mongodb://[^\\s]+",
3322
- flags: "g",
3323
- replacement: "[REDACTED_DB_URL]",
3324
- description: "MongoDB connection strings",
3325
- category: "db_url",
3326
- severity: "high"
3327
- },
3328
- {
3329
- source: "mysql://[^\\s]+",
3330
- flags: "g",
3331
- replacement: "[REDACTED_DB_URL]",
3332
- description: "MySQL connection strings",
3333
- category: "db_url",
3334
- severity: "high"
3335
- },
3336
- {
3337
- source: "AKIA[A-Z0-9]{16}",
3338
- flags: "g",
3339
- replacement: "[REDACTED_AWS_KEY]",
3340
- description: "AWS access keys",
3341
- category: "api_key",
3342
- severity: "critical"
3343
- },
3344
- {
3345
- source: "ghp_[a-zA-Z0-9]{36}",
3346
- flags: "g",
3347
- replacement: "[REDACTED_GH_TOKEN]",
3348
- description: "GitHub personal access tokens",
3349
- category: "token",
3350
- severity: "high"
3351
- },
3352
- {
3353
- source: "gho_[a-zA-Z0-9]{36}",
3354
- flags: "g",
3355
- replacement: "[REDACTED_GH_TOKEN]",
3356
- description: "GitHub OAuth tokens",
3357
- category: "token",
3358
- severity: "high"
3359
- },
3360
- {
3361
- source: "github_pat_[a-zA-Z0-9_]{22,}",
3362
- flags: "g",
3363
- replacement: "[REDACTED_GH_TOKEN]",
3364
- description: "GitHub fine-grained PATs",
3365
- category: "token",
3366
- severity: "high"
3367
- },
3368
- {
3369
- source: "cvk_[a-f0-9]{64}",
3370
- flags: "g",
3371
- replacement: "[REDACTED_CANDENGO_KEY]",
3372
- description: "Candengo API keys",
3373
- category: "api_key",
3374
- severity: "critical"
3375
- },
3376
- {
3377
- source: "xox[bpras]-[a-zA-Z0-9\\-]+",
3378
- flags: "g",
3379
- replacement: "[REDACTED_SLACK_TOKEN]",
3380
- description: "Slack tokens",
3381
- category: "token",
3382
- severity: "high"
3383
- }
3384
- ];
3385
- function compileCustomPatterns(patterns) {
3386
- const compiled = [];
3387
- for (const pattern of patterns) {
3388
- try {
3389
- new RegExp(pattern);
3390
- compiled.push({
3391
- source: pattern,
3392
- flags: "g",
3393
- replacement: "[REDACTED_CUSTOM]",
3394
- description: `Custom pattern: ${pattern}`,
3395
- category: "custom",
3396
- severity: "medium"
3397
- });
3398
- } catch {}
3399
- }
3400
- return compiled;
3401
- }
3402
- function scrubSecrets(text, customPatterns = []) {
3403
- let result = text;
3404
- const allPatterns = [...DEFAULT_PATTERNS, ...compileCustomPatterns(customPatterns)];
3405
- for (const pattern of allPatterns) {
3406
- result = result.replace(new RegExp(pattern.source, pattern.flags), pattern.replacement);
3407
- }
3408
- return result;
3409
- }
3410
- function containsSecrets(text, customPatterns = []) {
3411
- const allPatterns = [...DEFAULT_PATTERNS, ...compileCustomPatterns(customPatterns)];
3412
- for (const pattern of allPatterns) {
3413
- if (new RegExp(pattern.source, pattern.flags).test(text))
3414
- return true;
3415
- }
3416
- return false;
3417
- }
3418
-
3419
3537
  // src/capture/quality.ts
3420
3538
  var QUALITY_THRESHOLD = 0.1;
3421
3539
  function scoreQuality(input) {
@@ -3833,7 +3951,8 @@ async function saveObservation(db, config, input) {
3833
3951
  const factsJson = structuredFacts.length > 0 ? config.scrubbing.enabled ? scrubSecrets(JSON.stringify(structuredFacts), customPatterns) : JSON.stringify(structuredFacts) : null;
3834
3952
  const filesReadJson = filesRead ? JSON.stringify(filesRead) : null;
3835
3953
  const filesModifiedJson = filesModified ? JSON.stringify(filesModified) : null;
3836
- let sensitivity = input.sensitivity ?? config.scrubbing.default_sensitivity;
3954
+ const fleetProject = isFleetProjectName(project.name, config);
3955
+ let sensitivity = input.sensitivity ?? (fleetProject ? "shared" : config.scrubbing.default_sensitivity);
3837
3956
  if (config.scrubbing.enabled && containsSecrets([input.title, input.narrative, JSON.stringify(input.facts)].filter(Boolean).join(" "), customPatterns)) {
3838
3957
  if (sensitivity === "shared") {
3839
3958
  sensitivity = "personal";