ctxtop 0.0.0-stage → 4.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.antigravity-plugin/plugin.json +36 -0
- package/.claude-plugin/plugin.json +18 -0
- package/CHECKSUMS.sha256 +166 -0
- package/LICENSE +27 -0
- package/README.md +350 -2
- package/SECURITY.md +22 -0
- package/agents/ah-flash-scout.md +25 -0
- package/assets/advanced_view_loaded_420px.png +0 -0
- package/assets/hero-demo.svg +188 -0
- package/assets/logo.svg +19 -0
- package/assets/simple_view_lean_420px.png +0 -0
- package/assets/simple_view_loaded_420px.png +0 -0
- package/assets/social-preview.svg +72 -0
- package/assets/zone_c_step3_shield_full.png +0 -0
- package/bin/ah-statusline.mjs +175 -0
- package/bin/cli.mjs +3 -0
- package/bin/ctxtop.mjs +1799 -0
- package/commands/cost-radar.md +11 -0
- package/commands/evolve.md +10 -0
- package/commands/handoff.md +10 -0
- package/commands/horizon.md +10 -0
- package/commands/inspect.md +10 -0
- package/commands/precheck.md +10 -0
- package/config/default_flightcheck.json +133 -0
- package/config/modelCatalog.json +785 -0
- package/config/platformQuirks.json +19 -0
- package/hooks/handlers/compact-lifecycle.mjs +577 -0
- package/hooks/handlers/config-change.mjs +402 -0
- package/hooks/handlers/hookIo.mjs +311 -0
- package/hooks/handlers/hookIpcClient.mjs +83 -0
- package/hooks/handlers/post-model-switch.mjs +111 -0
- package/hooks/handlers/post-tool-use.mjs +366 -0
- package/hooks/handlers/pre-model-switch.mjs +1048 -0
- package/hooks/handlers/pre-tool-use.mjs +632 -0
- package/hooks/handlers/stop-gate.mjs +526 -0
- package/hooks/handlers/user-prompt-submit.mjs +367 -0
- package/hooks/hooks.json +131 -0
- package/hooks/post_invocation.mjs +13 -0
- package/hooks/post_tool_use.mjs +25 -0
- package/hooks/pre_invocation.mjs +14 -0
- package/hooks/pre_tool_use.mjs +97 -0
- package/hooks/register.js +726 -0
- package/hooks/session_start.mjs +24 -0
- package/hooks/stop.mjs +23 -0
- package/hooks.json +56 -0
- package/package.json +110 -4
- package/packages/antigravity/package.json +14 -0
- package/packages/antigravity/src/index.mjs +1 -0
- package/packages/claude-code/package.json +14 -0
- package/packages/claude-code/src/index.mjs +2 -0
- package/packages/core/package.json +14 -0
- package/packages/core/src/index.mjs +1 -0
- package/packages/engine-node/package.json +14 -0
- package/packages/engine-node/src/index.mjs +1 -0
- package/packages/mcp/package.json +17 -0
- package/packages/mcp/src/server.mjs +396 -0
- package/packages/pro/LICENSE-COMMERCIAL.md +19 -0
- package/packages/pro/README-SIGNING.md +30 -0
- package/packages/pro/package.json +16 -0
- package/packages/pro/src/index.mjs +3 -0
- package/packages/pro/src/team_finops_pack.mjs +241 -0
- package/packages/pro/workers/polar_webhook_worker.mjs +215 -0
- package/plugin.json +36 -0
- package/rules/AGENTS.md +10 -0
- package/sidecars/horizon-daemon/ipcServer.mjs +24 -0
- package/sidecars/horizon-daemon/server.mjs +2038 -0
- package/sidecars/horizon-daemon/sidecar.json +19 -0
- package/skills/cost-radar/SKILL.md +21 -0
- package/skills/evolve/SKILL.md +21 -0
- package/skills/handoff/SKILL.md +50 -0
- package/skills/horizon/SKILL.md +17 -0
- package/skills/inspect/SKILL.md +18 -0
- package/skills/precheck/SKILL.md +44 -0
- package/src/adapters/antigravity/index.mjs +109 -0
- package/src/adapters/claude-code/index.mjs +194 -0
- package/src/cli/cost_radar.mjs +70 -0
- package/src/cli/doctor.mjs +351 -0
- package/src/cli/horizon_hud.mjs +888 -0
- package/src/cli/init.mjs +425 -0
- package/src/cli/postmortem.mjs +515 -0
- package/src/coach/cacheGravityRouter.mjs +12 -0
- package/src/config/modelCatalog.json +704 -0
- package/src/config/platformQuirks.json +19 -0
- package/src/core/action_queue.mjs +264 -0
- package/src/core/ag_bridge.mjs +346 -0
- package/src/core/atomic_io.mjs +124 -0
- package/src/core/baseline.mjs +607 -0
- package/src/core/blindspot_sensors.mjs +399 -0
- package/src/core/bucket_b_runtime.mjs +1200 -0
- package/src/core/classifier.mjs +1757 -0
- package/src/core/coach/circuit_breakers.mjs +650 -0
- package/src/core/coach/contradiction_auditor.mjs +239 -0
- package/src/core/coach/effort_advisor.mjs +143 -0
- package/src/core/coach/handoff_hydrator.mjs +483 -0
- package/src/core/coach/progressive_checkpointer.mjs +551 -0
- package/src/core/coach/prompt_decomposer_coach.mjs +141 -0
- package/src/core/coach/reflector_queue.mjs +505 -0
- package/src/core/coach/saved_quota_ledger.mjs +615 -0
- package/src/core/coach/subagent_ipi_sanitizer.mjs +200 -0
- package/src/core/coach/subagent_phantom_auditor.mjs +261 -0
- package/src/core/coach/task_quota_router.mjs +955 -0
- package/src/core/coach/turn_zero_gate.mjs +574 -0
- package/src/core/compaction_coach.mjs +1126 -0
- package/src/core/config.mjs +214 -0
- package/src/core/constants.mjs +1987 -0
- package/src/core/detectors_det01_to_16.mjs +1523 -0
- package/src/core/doc_indexer.mjs +338 -0
- package/src/core/episodic_memory.mjs +135 -0
- package/src/core/fts5_output_compressor.mjs +359 -0
- package/src/core/git_ledger.mjs +189 -0
- package/src/core/handoff_audit.mjs +558 -0
- package/src/core/history.mjs +221 -0
- package/src/core/hook_diagnostics.mjs +158 -0
- package/src/core/horizon_classifier.mjs +326 -0
- package/src/core/horizon_config.mjs +802 -0
- package/src/core/host_watchdog.mjs +200 -0
- package/src/core/layer5_hygiene.mjs +1084 -0
- package/src/core/log_resolver.mjs +231 -0
- package/src/core/mcp_auditor.mjs +138 -0
- package/src/core/micro_skills_store.mjs +91 -0
- package/src/core/offenders.mjs +161 -0
- package/src/core/parser.mjs +1997 -0
- package/src/core/path_canonicalizer.mjs +73 -0
- package/src/core/paths.mjs +311 -0
- package/src/core/prefix_cache_shield.mjs +386 -0
- package/src/core/pro_license.mjs +385 -0
- package/src/core/prompt_rubric.mjs +518 -0
- package/src/core/pruner.mjs +433 -0
- package/src/core/quota.mjs +1860 -0
- package/src/core/rsi_instinct_engine.mjs +1197 -0
- package/src/core/rsl_clash_auditor.mjs +423 -0
- package/src/core/rsl_distiller.mjs +372 -0
- package/src/core/rsl_ledger.mjs +969 -0
- package/src/core/rsl_triage.mjs +909 -0
- package/src/core/sanitizer.mjs +121 -0
- package/src/core/session_auditor.mjs +658 -0
- package/src/core/session_forensics.mjs +303 -0
- package/src/core/session_timeline.mjs +358 -0
- package/src/core/settings.mjs +99 -0
- package/src/core/shared/detectors.mjs +1342 -0
- package/src/core/shared/fnv1a64.mjs +85 -0
- package/src/core/shared/handoff_schema.mjs +123 -0
- package/src/core/shared/index.mjs +5 -0
- package/src/core/shared/ipi_sanitizer.mjs +165 -0
- package/src/core/shared/micro_skills.mjs +1 -0
- package/src/core/shared/plugin_validator.mjs +202 -0
- package/src/core/shared/prompt_hygiene_governor.mjs +1 -0
- package/src/core/shield_manager.mjs +1417 -0
- package/src/core/sidecar_store.mjs +302 -0
- package/src/core/sprint_metrics.mjs +366 -0
- package/src/core/sqlite_client.mjs +283 -0
- package/src/core/storage_doctor.mjs +358 -0
- package/src/core/subagents.mjs +609 -0
- package/src/core/swarm_topology.mjs +360 -0
- package/src/core/task_quota_router.mjs +1656 -0
- package/src/core/telemetry/cache_ttl_watchdog.mjs +284 -0
- package/src/core/telemetry/scope_lock_registry.mjs +578 -0
- package/src/core/telemetry/shadow_file_detector.mjs +291 -0
- package/src/core/telemetry/trajectory_watcher.mjs +259 -0
- package/src/core/timeline_svg.mjs +236 -0
- package/src/core/tool_surgeon.mjs +305 -0
- package/src/core/trajectory_stitcher.mjs +645 -0
- package/src/core/u_shaped_optimizer.mjs +789 -0
- package/src/core/wal_hygiene.mjs +233 -0
- package/src/core-isomorphic/catalog/detectorCatalog.mjs +558 -0
- package/src/core-isomorphic/catalog/index.mjs +1 -0
- package/src/core-isomorphic/coach/cacheGravityRouter.mjs +1547 -0
- package/src/core-isomorphic/coach/microSkills.mjs +1 -0
- package/src/core-isomorphic/coach/promptHygieneGovernor.mjs +1 -0
- package/src/core-isomorphic/coach/repoCalibration.mjs +254 -0
- package/src/core-isomorphic/coach/resumeToolsetAuditor.mjs +321 -0
- package/src/core-isomorphic/coach/savedQuotaLedger.mjs +527 -0
- package/src/core-isomorphic/coach/smartCompactAdvisor.mjs +305 -0
- package/src/core-isomorphic/coach/stateBatonHydrator.mjs +810 -0
- package/src/core-isomorphic/coach/ttlWatchdog.mjs +464 -0
- package/src/core-isomorphic/config/anthropicModelCatalog.json +606 -0
- package/src/core-isomorphic/config/catalogLoader.mjs +1268 -0
- package/src/core-isomorphic/config/horizonConfig.mjs +1055 -0
- package/src/core-isomorphic/control/callTree.mjs +338 -0
- package/src/core-isomorphic/control/coachLatches.mjs +126 -0
- package/src/core-isomorphic/control/controlStateMachine.mjs +205 -0
- package/src/core-isomorphic/control/index.mjs +49 -0
- package/src/core-isomorphic/control/swarmWorktree.mjs +109 -0
- package/src/core-isomorphic/control/toolSuspension.mjs +197 -0
- package/src/core-isomorphic/diagnostics/drift_guard.mjs +382 -0
- package/src/core-isomorphic/diagnostics/index.mjs +9 -0
- package/src/core-isomorphic/diagnostics/micro_skills.mjs +471 -0
- package/src/core-isomorphic/diagnostics/prompt_hygiene_governor.mjs +888 -0
- package/src/core-isomorphic/economics/billingBlocks.mjs +251 -0
- package/src/core-isomorphic/economics/cacheGravity.mjs +369 -0
- package/src/core-isomorphic/economics/cliffBreaker.mjs +256 -0
- package/src/core-isomorphic/economics/driftLatency.mjs +433 -0
- package/src/core-isomorphic/economics/dualTtl.mjs +198 -0
- package/src/core-isomorphic/economics/index.mjs +47 -0
- package/src/core-isomorphic/economics/retentionRadar.mjs +348 -0
- package/src/core-isomorphic/guards/bucketBAlphaBeta.mjs +860 -0
- package/src/core-isomorphic/guards/bucketBGamma.mjs +442 -0
- package/src/core-isomorphic/guards/bucketBHeuristics.mjs +37 -0
- package/src/core-isomorphic/guards/circuitBreakers.mjs +1202 -0
- package/src/core-isomorphic/guards/det01_03_shell_secret.mjs +1184 -0
- package/src/core-isomorphic/guards/det04_12_thrash_rot.mjs +572 -0
- package/src/core-isomorphic/guards/det13_20_mcp_scope.mjs +913 -0
- package/src/core-isomorphic/guards/docIndexCore.mjs +446 -0
- package/src/core-isomorphic/guards/losslessPostSanitizer.mjs +317 -0
- package/src/core-isomorphic/guards/slopsquatGate.mjs +551 -0
- package/src/core-isomorphic/index.js +27 -0
- package/src/core-isomorphic/telemetry/otelExporter.mjs +187 -0
- package/src/core-isomorphic/telemetry/scopeLockRegistry.mjs +522 -0
- package/src/core-isomorphic/telemetry/sprintMetrics.mjs +327 -0
- package/src/core-isomorphic/telemetry/swarmTopology.mjs +152 -0
- package/src/core-isomorphic/telemetry/telemetryBridge.mjs +1156 -0
- package/src/core-isomorphic/ui/brailleSparkline.mjs +142 -0
- package/src/core-isomorphic/ui/costRadarSvg.mjs +276 -0
- package/src/core-isomorphic/ui/horizonGrid.mjs +475 -0
- package/src/core-isomorphic/ui/horizonPanelHtml.mjs +1237 -0
- package/src/core-isomorphic/ui/modViews.mjs +1504 -0
- package/src/core-isomorphic/ui/pdfReceiptGenerator.mjs +329 -0
- package/src/core-isomorphic/ustf/index.mjs +7 -0
- package/src/core-isomorphic/ustf/normalizer.mjs +512 -0
- package/src/core-isomorphic/ustf/schema.mjs +186 -0
- package/src/core-isomorphic/utils/runtime.mjs +962 -0
- package/src/demo/synthetic_states.mjs +661 -0
- package/src/diagnostics/bugReportGenerator.mjs +253 -0
- package/src/diagnostics/claudeHookCapabilities.mjs +396 -0
- package/src/diagnostics/doctorEngine.mjs +295 -0
- package/src/engine-node/db/index.mjs +58 -0
- package/src/engine-node/db/jsonl_fallback.mjs +184 -0
- package/src/engine-node/db/probe.mjs +342 -0
- package/src/engine-node/db/sqlite_wrapper.mjs +122 -0
- package/src/engine-node/index.mjs +12 -0
- package/src/engine-node/ipc/client.mjs +447 -0
- package/src/engine-node/ipc/index.mjs +22 -0
- package/src/engine-node/ipc/protocol.mjs +177 -0
- package/src/engine-node/ipc/server.mjs +445 -0
- package/src/engine-node/offloader/index.mjs +6 -0
- package/src/engine-node/offloader/tool_offloader.mjs +241 -0
- package/src/engine-node/sensors/blindspots.mjs +376 -0
- package/src/engine-node/sensors/git_merge_tree.mjs +223 -0
- package/src/engine-node/sensors/index.mjs +7 -0
- package/src/hooks/antigravity_rsl_hook.mjs +250 -0
- package/src/hooks/claude_rsl_hook.mjs +288 -0
- package/src/hooks/portfolio_runtime.mjs +858 -0
- package/src/hooks/post_invocation.mjs +611 -0
- package/src/hooks/pre_invocation.mjs +1725 -0
- package/src/hooks/session_start.mjs +199 -0
- package/src/shims/agy_cli_compressor.mjs +705 -0
- package/src/shims/agy_git_guard.mjs +749 -0
- package/src/shims/ah_log_query.mjs +39 -0
- package/src/ui/costRadarSvg.mjs +6 -0
- package/src/ui/horizonGrid.mjs +1 -0
- package/src/ui/horizonPanelHtml.mjs +1 -0
- package/src/ui/horizon_panel.css +167 -0
- package/src/ui/horizon_panel.js +212 -0
- package/src/ui/modViews.mjs +1 -0
- package/src/utils/modelSwitchTelemetry.mjs +308 -0
- package/src/utils/nodeIo.mjs +59 -0
- package/src/utils/runtime.mjs +1 -0
- package/src/utils/sessionCalibration.mjs +657 -0
- package/src/webview/advanced_view.html +1808 -0
- package/src/webview/horizon_panel.html +12397 -0
- package/src/webview/registry.mjs +80 -0
- package/src/webview/simple_view.html +1370 -0
|
@@ -0,0 +1,446 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* @file src/guards/docIndexCore.mjs
|
|
3
|
+
* Pure JavaScript zero-token structural document & log indexer (`< 20ms`, 0 LLM tokens).
|
|
4
|
+
*
|
|
5
|
+
* Architectural Invariants:
|
|
6
|
+
* - ZERO `node:*` imports, ZERO `require()`, ZERO dynamic ESM imports (Mod Law 4 compliant).
|
|
7
|
+
* - Strictly excludes `ctxvault` SKU paths (`packages/airlock/`, `airlock-schema-canary.yml`,
|
|
8
|
+
* `docs/HIGHBEAM_AIRLOCK_*.md`).
|
|
9
|
+
* - Scans outside fenced code blocks for Markdown headings (`#`..`####`), code definitions
|
|
10
|
+
* (`export`, `function`, `class`, `interface`, `type`, `def`, `async def`, `func`, `struct`),
|
|
11
|
+
* or top-level JSON/YAML keys, with a contiguous 150-line unstructured fallback when `< 3`
|
|
12
|
+
* structural boundaries exist in files `> 400` lines.
|
|
13
|
+
*/
|
|
14
|
+
|
|
15
|
+
import { normalizeSlashes, sha256Hex } from '../utils/runtime.mjs';
|
|
16
|
+
|
|
17
|
+
export const UNSTRUCTURED_FALLBACK_LINE_THRESHOLD = 400;
|
|
18
|
+
export const UNSTRUCTURED_SPAN_SIZE = 150;
|
|
19
|
+
export const MIN_STRUCTURAL_BOUNDARIES = 3;
|
|
20
|
+
export const MAX_IDENTIFIERS_PER_SPAN = 8;
|
|
21
|
+
|
|
22
|
+
const STOPWORDS = new Set([
|
|
23
|
+
'const', 'let', 'var', 'function', 'async', 'await', 'return', 'export', 'import', 'from',
|
|
24
|
+
'default', 'class', 'interface', 'type', 'extends', 'implements', 'public', 'private',
|
|
25
|
+
'protected', 'static', 'true', 'false', 'null', 'undefined', 'this', 'super', 'new',
|
|
26
|
+
'throw', 'catch', 'try', 'finally', 'else', 'switch', 'case', 'break', 'continue',
|
|
27
|
+
'while', 'for', 'typeof', 'instanceof', 'void', 'delete', 'with', 'yield', 'def',
|
|
28
|
+
'self', 'none', 'pass', 'elif', 'lambda', 'func', 'struct', 'impl', 'pub', 'mut',
|
|
29
|
+
'string', 'number', 'boolean', 'object', 'array', 'promise', 'record', 'the', 'and',
|
|
30
|
+
'with', 'that', 'this', 'from', 'into', 'when', 'where', 'then', 'only', 'also',
|
|
31
|
+
'must', 'should', 'never', 'always', 'line', 'lines', 'file', 'files', 'test', 'data'
|
|
32
|
+
]);
|
|
33
|
+
|
|
34
|
+
/**
|
|
35
|
+
* Checks whether a path belongs to the isolated sister security SKU and must never be indexed or read.
|
|
36
|
+
*
|
|
37
|
+
* @param {string} [filePath='']
|
|
38
|
+
* @returns {boolean}
|
|
39
|
+
*/
|
|
40
|
+
export function isForbiddenSkuPath(filePath = '') {
|
|
41
|
+
const norm = normalizeSlashes(String(filePath || '')).trim();
|
|
42
|
+
if (!norm) return false;
|
|
43
|
+
const lower = norm.toLowerCase();
|
|
44
|
+
if (
|
|
45
|
+
/(?:^|\/)packages\/airlock(?:\/|$)/i.test(lower) ||
|
|
46
|
+
lower.endsWith('.github/workflows/airlock-schema-canary.yml') ||
|
|
47
|
+
/(?:^|\/)\.github\/workflows\/airlock-schema-canary\.yml$/i.test(lower) ||
|
|
48
|
+
/(?:^|\/)docs\/highbeam_airlock_[^/]*\.md$/i.test(lower)
|
|
49
|
+
) {
|
|
50
|
+
return true;
|
|
51
|
+
}
|
|
52
|
+
return false;
|
|
53
|
+
}
|
|
54
|
+
|
|
55
|
+
/**
|
|
56
|
+
* Splits text into lines across CRLF and LF line endings.
|
|
57
|
+
*
|
|
58
|
+
* @param {string} [rawText='']
|
|
59
|
+
* @returns {string[]}
|
|
60
|
+
*/
|
|
61
|
+
export function splitLinesNormalized(rawText = '') {
|
|
62
|
+
const str = String(rawText ?? '');
|
|
63
|
+
if (str.length === 0) return [];
|
|
64
|
+
return str.replace(/\r\n/g, '\n').replace(/\r/g, '\n').split('\n');
|
|
65
|
+
}
|
|
66
|
+
|
|
67
|
+
/**
|
|
68
|
+
* Parses YAML frontmatter from a `.index.md` document.
|
|
69
|
+
*
|
|
70
|
+
* @param {string} [markdownText='']
|
|
71
|
+
* @returns {{ indexed_file: string, sha256: string, total_lines: number, generated_by: string } | null}
|
|
72
|
+
*/
|
|
73
|
+
export function parseIndexFrontmatter(markdownText = '') {
|
|
74
|
+
const lines = splitLinesNormalized(markdownText);
|
|
75
|
+
if (lines.length < 3 || lines[0].trim() !== '---') return null;
|
|
76
|
+
const meta = {
|
|
77
|
+
indexed_file: '',
|
|
78
|
+
sha256: '',
|
|
79
|
+
total_lines: 0,
|
|
80
|
+
generated_by: ''
|
|
81
|
+
};
|
|
82
|
+
let closed = false;
|
|
83
|
+
for (let i = 1; i < Math.min(lines.length, 25); i++) {
|
|
84
|
+
const trimmed = lines[i].trim();
|
|
85
|
+
if (trimmed === '---') {
|
|
86
|
+
closed = true;
|
|
87
|
+
break;
|
|
88
|
+
}
|
|
89
|
+
const sepIdx = trimmed.indexOf(':');
|
|
90
|
+
if (sepIdx === -1) continue;
|
|
91
|
+
const key = trimmed.slice(0, sepIdx).trim();
|
|
92
|
+
const val = trimmed.slice(sepIdx + 1).trim().replace(/^["']|["']$/g, '');
|
|
93
|
+
if (key === 'indexed_file') meta.indexed_file = val;
|
|
94
|
+
else if (key === 'sha256') meta.sha256 = val;
|
|
95
|
+
else if (key === 'total_lines') meta.total_lines = Number(val) || 0;
|
|
96
|
+
else if (key === 'generated_by') meta.generated_by = val;
|
|
97
|
+
}
|
|
98
|
+
return closed && meta.sha256 ? meta : null;
|
|
99
|
+
}
|
|
100
|
+
|
|
101
|
+
/**
|
|
102
|
+
* Compares live SHA-256 of a file against `.index.md` YAML frontmatter.
|
|
103
|
+
* Returns `0` (`FRESH`), `1` (`STALE`), or `2` (`MISSING`).
|
|
104
|
+
*
|
|
105
|
+
* @param {string} liveSha256
|
|
106
|
+
* @param {string | null | undefined} indexMarkdownText
|
|
107
|
+
* @returns {{ code: 0 | 1 | 2, status: 'FRESH' | 'STALE' | 'MISSING', recordedSha256: string | null }}
|
|
108
|
+
*/
|
|
109
|
+
export function compareIndexFreshness(liveSha256, indexMarkdownText) {
|
|
110
|
+
if (!indexMarkdownText || typeof indexMarkdownText !== 'string') {
|
|
111
|
+
return { code: 2, status: 'MISSING', recordedSha256: null };
|
|
112
|
+
}
|
|
113
|
+
const fm = parseIndexFrontmatter(indexMarkdownText);
|
|
114
|
+
if (!fm || !fm.sha256) {
|
|
115
|
+
return { code: 2, status: 'MISSING', recordedSha256: null };
|
|
116
|
+
}
|
|
117
|
+
if (String(fm.sha256).toLowerCase() === String(liveSha256 || '').toLowerCase()) {
|
|
118
|
+
return { code: 0, status: 'FRESH', recordedSha256: fm.sha256 };
|
|
119
|
+
}
|
|
120
|
+
return { code: 1, status: 'STALE', recordedSha256: fm.sha256 };
|
|
121
|
+
}
|
|
122
|
+
|
|
123
|
+
function sanitizeTableCell(text = '') {
|
|
124
|
+
return String(text || '')
|
|
125
|
+
.replace(/\|/g, '\\|')
|
|
126
|
+
.replace(/\s+/g, ' ')
|
|
127
|
+
.trim()
|
|
128
|
+
.slice(0, 96);
|
|
129
|
+
}
|
|
130
|
+
|
|
131
|
+
/**
|
|
132
|
+
* Extracts up to `maxCount` backticked terms or high-signal identifiers from a slice of lines.
|
|
133
|
+
*
|
|
134
|
+
* @param {string[]} lines
|
|
135
|
+
* @param {number} startIdx - 0-based inclusive
|
|
136
|
+
* @param {number} endIdx - 0-based inclusive
|
|
137
|
+
* @param {number} [maxCount=MAX_IDENTIFIERS_PER_SPAN]
|
|
138
|
+
* @returns {string[]}
|
|
139
|
+
*/
|
|
140
|
+
export function extractSpanIdentifiers(lines, startIdx, endIdx, maxCount = MAX_IDENTIFIERS_PER_SPAN) {
|
|
141
|
+
const counts = new Map();
|
|
142
|
+
const backtickTerms = [];
|
|
143
|
+
const backtickSeen = new Set();
|
|
144
|
+
|
|
145
|
+
const upper = Math.min(lines.length - 1, endIdx);
|
|
146
|
+
for (let i = Math.max(0, startIdx); i <= upper; i++) {
|
|
147
|
+
const line = lines[i] || '';
|
|
148
|
+
if (line.length > 500) continue;
|
|
149
|
+
|
|
150
|
+
// 1. Backticked terms first (high signal in Markdown/JSDoc/logs)
|
|
151
|
+
const btRegex = /`([A-Za-z0-9_./:-]{3,48})`/g;
|
|
152
|
+
let btMatch;
|
|
153
|
+
while ((btMatch = btRegex.exec(line)) !== null) {
|
|
154
|
+
const term = btMatch[1];
|
|
155
|
+
if (!backtickSeen.has(term) && !STOPWORDS.has(term.toLowerCase())) {
|
|
156
|
+
backtickSeen.add(term);
|
|
157
|
+
backtickTerms.push(term);
|
|
158
|
+
if (backtickTerms.length >= maxCount) break;
|
|
159
|
+
}
|
|
160
|
+
}
|
|
161
|
+
|
|
162
|
+
// 2. Identifiers (camelCase, PascalCase, snake_case, SCREAMING_SNAKE, error codes)
|
|
163
|
+
const idRegex = /\b([A-Za-z_][A-Za-z0-9_]{3,44})\b/g;
|
|
164
|
+
let idMatch;
|
|
165
|
+
while ((idMatch = idRegex.exec(line)) !== null) {
|
|
166
|
+
const ident = idMatch[1];
|
|
167
|
+
const lower = ident.toLowerCase();
|
|
168
|
+
if (STOPWORDS.has(lower)) continue;
|
|
169
|
+
const isStructuredIdent =
|
|
170
|
+
/[A-Z]/.test(ident) || ident.includes('_') || ident.length >= 6;
|
|
171
|
+
if (!isStructuredIdent) continue;
|
|
172
|
+
counts.set(ident, (counts.get(ident) || 0) + 1);
|
|
173
|
+
}
|
|
174
|
+
}
|
|
175
|
+
|
|
176
|
+
const out = [...backtickTerms.slice(0, maxCount)];
|
|
177
|
+
if (out.length < maxCount) {
|
|
178
|
+
const ranked = Array.from(counts.entries())
|
|
179
|
+
.filter(([ident]) => !backtickSeen.has(ident))
|
|
180
|
+
.sort((a, b) => b[1] - a[1] || a[0].localeCompare(b[0]));
|
|
181
|
+
for (const [ident] of ranked) {
|
|
182
|
+
out.push(ident);
|
|
183
|
+
if (out.length >= maxCount) break;
|
|
184
|
+
}
|
|
185
|
+
}
|
|
186
|
+
return out;
|
|
187
|
+
}
|
|
188
|
+
|
|
189
|
+
/**
|
|
190
|
+
* Detects structural boundary label on a single line (outside Markdown code fences).
|
|
191
|
+
*
|
|
192
|
+
* @param {string} line
|
|
193
|
+
* @param {boolean} isMarkdownLike
|
|
194
|
+
* @returns {string | null}
|
|
195
|
+
*/
|
|
196
|
+
function matchStructuralBoundary(line, isMarkdownLike) {
|
|
197
|
+
const trimmed = line.trim();
|
|
198
|
+
if (!trimmed || trimmed.length > 240) return null;
|
|
199
|
+
|
|
200
|
+
// Markdown headings (# .. ####)
|
|
201
|
+
const mdMatch = /^(#{1,4})\s+(.+)$/.exec(line);
|
|
202
|
+
if (mdMatch) {
|
|
203
|
+
return `${mdMatch[1]} ${mdMatch[2].trim().slice(0, 72)}`;
|
|
204
|
+
}
|
|
205
|
+
if (isMarkdownLike) return null;
|
|
206
|
+
|
|
207
|
+
// JS/TS exports, functions, classes, interfaces, types
|
|
208
|
+
const jsMatch =
|
|
209
|
+
/^(?:export\s+(?:default\s+)?)?(?:async\s+)?(?:function\*?\s+([A-Za-z_$][A-Za-z0-9_$]*)|class\s+([A-Za-z_$][A-Za-z0-9_$]*)|interface\s+([A-Za-z_$][A-Za-z0-9_$]*)|type\s+([A-Za-z_$][A-Za-z0-9_$]*)\s*=|const\s+([A-Za-z_$][A-Za-z0-9_$]*)\s*=\s*(?:async\s*)?(?:\([^\x29]*\)|[A-Za-z_$][A-Za-z0-9_$]*)\s*=>|(?:export\s+const\s+([A-Za-z_$][A-Za-z0-9_$]*)\s*=))/.exec(
|
|
210
|
+
line
|
|
211
|
+
);
|
|
212
|
+
if (jsMatch) {
|
|
213
|
+
const sym = jsMatch[1] || jsMatch[2] || jsMatch[3] || jsMatch[4] || jsMatch[5] || jsMatch[6];
|
|
214
|
+
if (sym) return sym;
|
|
215
|
+
}
|
|
216
|
+
|
|
217
|
+
// Python / Go / Rust definitions
|
|
218
|
+
const pyGoRustMatch =
|
|
219
|
+
/^(?:async\s+def|def|class|func(?:\s*\([^\x29]+\))?|pub\s+(?:async\s+)?fn|fn|pub\s+struct|struct|impl)\s+([A-Za-z_][A-Za-z0-9_]*)/.exec(
|
|
220
|
+
line
|
|
221
|
+
);
|
|
222
|
+
if (pyGoRustMatch) {
|
|
223
|
+
return pyGoRustMatch[1];
|
|
224
|
+
}
|
|
225
|
+
|
|
226
|
+
// Top-level JSON or YAML keys (0-2 leading spaces)
|
|
227
|
+
const jsonKeyMatch = /^ {0,2}"([A-Za-z0-9_.-]{2,48})"\s*:\s*[\x7b\x5b]/.exec(line);
|
|
228
|
+
if (jsonKeyMatch) {
|
|
229
|
+
return `"${jsonKeyMatch[1]}"`;
|
|
230
|
+
}
|
|
231
|
+
const yamlKeyMatch = /^([A-Za-z][A-Za-z0-9_.-]{1,48}):\s*(?:#.*)?$/.exec(line);
|
|
232
|
+
if (yamlKeyMatch && !/^(?:https?|file|mailto)$/i.test(yamlKeyMatch[1])) {
|
|
233
|
+
return `${yamlKeyMatch[1]}:`;
|
|
234
|
+
}
|
|
235
|
+
|
|
236
|
+
return null;
|
|
237
|
+
}
|
|
238
|
+
|
|
239
|
+
/**
|
|
240
|
+
* Builds deterministic zero-token `.index.md` content for any text/code/log file.
|
|
241
|
+
*
|
|
242
|
+
* @param {object} params
|
|
243
|
+
* @param {string} params.filePath
|
|
244
|
+
* @param {string} [params.displayPath]
|
|
245
|
+
* @param {string} params.rawText
|
|
246
|
+
* @param {string} [params.sha256]
|
|
247
|
+
* @returns {{
|
|
248
|
+
* markdown: string,
|
|
249
|
+
* sha256: string,
|
|
250
|
+
* totalLines: number,
|
|
251
|
+
* spanCount: number,
|
|
252
|
+
* usedFallback: boolean,
|
|
253
|
+
* indexTokens: number
|
|
254
|
+
* }}
|
|
255
|
+
*/
|
|
256
|
+
export function buildStructuralIndexMarkdown({
|
|
257
|
+
filePath = '',
|
|
258
|
+
displayPath = '',
|
|
259
|
+
rawText = '',
|
|
260
|
+
sha256 = ''
|
|
261
|
+
} = {}) {
|
|
262
|
+
const normalizedRaw = String(rawText ?? '').replace(/\r\n/g, '\n');
|
|
263
|
+
const computedSha = sha256 || sha256Hex(normalizedRaw);
|
|
264
|
+
const lines = splitLinesNormalized(rawText);
|
|
265
|
+
const totalLines = lines.length;
|
|
266
|
+
const labelPath = normalizeSlashes(displayPath || filePath || 'unknown');
|
|
267
|
+
const isMarkdownLike = /\.(?:md|mdx|markdown)$/i.test(labelPath);
|
|
268
|
+
|
|
269
|
+
const boundaries = [];
|
|
270
|
+
let inFence = false;
|
|
271
|
+
let fenceMarker = '';
|
|
272
|
+
|
|
273
|
+
for (let i = 0; i < totalLines; i++) {
|
|
274
|
+
const line = lines[i];
|
|
275
|
+
const fenceMatch = /^\s*(`{3,}|~{3,})/.exec(line);
|
|
276
|
+
if (fenceMatch) {
|
|
277
|
+
const marker = fenceMatch[1][0];
|
|
278
|
+
if (!inFence) {
|
|
279
|
+
inFence = true;
|
|
280
|
+
fenceMarker = marker;
|
|
281
|
+
} else if (marker === fenceMarker) {
|
|
282
|
+
inFence = false;
|
|
283
|
+
fenceMarker = '';
|
|
284
|
+
}
|
|
285
|
+
continue;
|
|
286
|
+
}
|
|
287
|
+
if (inFence) continue;
|
|
288
|
+
|
|
289
|
+
const label = matchStructuralBoundary(line, isMarkdownLike);
|
|
290
|
+
if (label) {
|
|
291
|
+
boundaries.push({ lineNum: i + 1, label });
|
|
292
|
+
}
|
|
293
|
+
}
|
|
294
|
+
|
|
295
|
+
let spans = [];
|
|
296
|
+
let usedFallback = false;
|
|
297
|
+
|
|
298
|
+
if (boundaries.length < MIN_STRUCTURAL_BOUNDARIES && totalLines > UNSTRUCTURED_FALLBACK_LINE_THRESHOLD) {
|
|
299
|
+
usedFallback = true;
|
|
300
|
+
for (let start = 1; start <= totalLines; start += UNSTRUCTURED_SPAN_SIZE) {
|
|
301
|
+
const end = Math.min(totalLines, start + UNSTRUCTURED_SPAN_SIZE - 1);
|
|
302
|
+
const idents = extractSpanIdentifiers(lines, start - 1, end - 1);
|
|
303
|
+
spans.push({
|
|
304
|
+
startLine: start,
|
|
305
|
+
endLine: end,
|
|
306
|
+
label: `Unstructured Span L${start}-L${end}`,
|
|
307
|
+
identifiers: idents
|
|
308
|
+
});
|
|
309
|
+
}
|
|
310
|
+
} else if (boundaries.length > 0) {
|
|
311
|
+
// Include leading preamble if boundary 0 starts after line 1
|
|
312
|
+
if (boundaries[0].lineNum > 1) {
|
|
313
|
+
const preEnd = boundaries[0].lineNum - 1;
|
|
314
|
+
spans.push({
|
|
315
|
+
startLine: 1,
|
|
316
|
+
endLine: preEnd,
|
|
317
|
+
label: 'Preamble / Imports',
|
|
318
|
+
identifiers: extractSpanIdentifiers(lines, 0, preEnd - 1)
|
|
319
|
+
});
|
|
320
|
+
}
|
|
321
|
+
for (let idx = 0; idx < boundaries.length; idx++) {
|
|
322
|
+
const curr = boundaries[idx];
|
|
323
|
+
const next = boundaries[idx + 1];
|
|
324
|
+
const endLine = next ? Math.max(curr.lineNum, next.lineNum - 1) : totalLines;
|
|
325
|
+
spans.push({
|
|
326
|
+
startLine: curr.lineNum,
|
|
327
|
+
endLine,
|
|
328
|
+
label: curr.label,
|
|
329
|
+
identifiers: extractSpanIdentifiers(lines, curr.lineNum - 1, endLine - 1)
|
|
330
|
+
});
|
|
331
|
+
}
|
|
332
|
+
} else {
|
|
333
|
+
const end = Math.max(1, totalLines);
|
|
334
|
+
spans.push({
|
|
335
|
+
startLine: 1,
|
|
336
|
+
endLine: end,
|
|
337
|
+
label: totalLines === 0 ? 'Empty File' : `Full Span L1-L${end}`,
|
|
338
|
+
identifiers: totalLines === 0 ? [] : extractSpanIdentifiers(lines, 0, end - 1)
|
|
339
|
+
});
|
|
340
|
+
}
|
|
341
|
+
|
|
342
|
+
// Cap extreme table rows at 120 spans by coalescing adjacent micro-spans if needed
|
|
343
|
+
if (spans.length > 120) {
|
|
344
|
+
const bucketSize = Math.ceil(spans.length / 80);
|
|
345
|
+
const merged = [];
|
|
346
|
+
for (let i = 0; i < spans.length; i += bucketSize) {
|
|
347
|
+
const group = spans.slice(i, i + bucketSize);
|
|
348
|
+
const combinedIdents = [];
|
|
349
|
+
const seen = new Set();
|
|
350
|
+
for (const g of group) {
|
|
351
|
+
for (const id of g.identifiers) {
|
|
352
|
+
if (!seen.has(id)) {
|
|
353
|
+
seen.add(id);
|
|
354
|
+
combinedIdents.push(id);
|
|
355
|
+
}
|
|
356
|
+
if (combinedIdents.length >= MAX_IDENTIFIERS_PER_SPAN) break;
|
|
357
|
+
}
|
|
358
|
+
}
|
|
359
|
+
merged.push({
|
|
360
|
+
startLine: group[0].startLine,
|
|
361
|
+
endLine: group[group.length - 1].endLine,
|
|
362
|
+
label: group.length > 1 ? `${group[0].label} (+${group.length - 1} more)` : group[0].label,
|
|
363
|
+
identifiers: combinedIdents
|
|
364
|
+
});
|
|
365
|
+
}
|
|
366
|
+
spans = merged;
|
|
367
|
+
}
|
|
368
|
+
|
|
369
|
+
const rows = spans.map((sp) => {
|
|
370
|
+
const rangeStr = `L${sp.startLine}-L${sp.endLine}`;
|
|
371
|
+
const sectionStr = sanitizeTableCell(sp.label);
|
|
372
|
+
const identStr =
|
|
373
|
+
sp.identifiers.length > 0
|
|
374
|
+
? sp.identifiers.map((id) => `\`${sanitizeTableCell(id)}\``).join(', ')
|
|
375
|
+
: '—';
|
|
376
|
+
return `| ${rangeStr} | ${sectionStr} | ${identStr} |`;
|
|
377
|
+
});
|
|
378
|
+
|
|
379
|
+
const markdown = [
|
|
380
|
+
'---',
|
|
381
|
+
`indexed_file: "${labelPath}"`,
|
|
382
|
+
`sha256: "${computedSha}"`,
|
|
383
|
+
`total_lines: ${totalLines}`,
|
|
384
|
+
'generated_by: "ctxtop-doc-indexer"',
|
|
385
|
+
'---',
|
|
386
|
+
'',
|
|
387
|
+
`# Structural Index: \`${labelPath}\` (${totalLines} lines)`,
|
|
388
|
+
'',
|
|
389
|
+
'| Lines | Section / Symbol | High-Signal Identifiers |',
|
|
390
|
+
'| :--- | :--- | :--- |',
|
|
391
|
+
...rows,
|
|
392
|
+
''
|
|
393
|
+
].join('\n');
|
|
394
|
+
|
|
395
|
+
return {
|
|
396
|
+
markdown,
|
|
397
|
+
sha256: computedSha,
|
|
398
|
+
totalLines,
|
|
399
|
+
spanCount: spans.length,
|
|
400
|
+
usedFallback,
|
|
401
|
+
indexTokens: Math.max(1, Math.ceil(markdown.length / 4))
|
|
402
|
+
};
|
|
403
|
+
}
|
|
404
|
+
|
|
405
|
+
/**
|
|
406
|
+
* Formats 1-indexed line-numbered slice output (` 142 | ...`) across CRLF and LF inputs.
|
|
407
|
+
*
|
|
408
|
+
* @param {string} [rawText='']
|
|
409
|
+
* @param {number} [startLine=1]
|
|
410
|
+
* @param {number} [endLine=1]
|
|
411
|
+
* @returns {{ text: string, startLine: number, endLine: number, totalLines: number }}
|
|
412
|
+
*/
|
|
413
|
+
export function formatSlicedLines(rawText = '', startLine = 1, endLine = 1) {
|
|
414
|
+
const lines = splitLinesNormalized(rawText);
|
|
415
|
+
const totalLines = lines.length;
|
|
416
|
+
if (totalLines === 0) {
|
|
417
|
+
return { text: '', startLine: 0, endLine: 0, totalLines: 0 };
|
|
418
|
+
}
|
|
419
|
+
const s = Math.max(1, Math.min(totalLines, Math.floor(Number(startLine) || 1)));
|
|
420
|
+
const e = Math.max(s, Math.min(totalLines, Math.floor(Number(endLine) || s)));
|
|
421
|
+
const width = Math.max(6, String(e).length);
|
|
422
|
+
const out = [];
|
|
423
|
+
for (let lineNum = s; lineNum <= e; lineNum++) {
|
|
424
|
+
out.push(`${String(lineNum).padStart(width, ' ')} | ${lines[lineNum - 1]}`);
|
|
425
|
+
}
|
|
426
|
+
return {
|
|
427
|
+
text: out.join('\n'),
|
|
428
|
+
startLine: s,
|
|
429
|
+
endLine: e,
|
|
430
|
+
totalLines
|
|
431
|
+
};
|
|
432
|
+
}
|
|
433
|
+
|
|
434
|
+
export default {
|
|
435
|
+
UNSTRUCTURED_FALLBACK_LINE_THRESHOLD,
|
|
436
|
+
UNSTRUCTURED_SPAN_SIZE,
|
|
437
|
+
MIN_STRUCTURAL_BOUNDARIES,
|
|
438
|
+
MAX_IDENTIFIERS_PER_SPAN,
|
|
439
|
+
isForbiddenSkuPath,
|
|
440
|
+
splitLinesNormalized,
|
|
441
|
+
parseIndexFrontmatter,
|
|
442
|
+
compareIndexFreshness,
|
|
443
|
+
extractSpanIdentifiers,
|
|
444
|
+
buildStructuralIndexMarkdown,
|
|
445
|
+
formatSlicedLines
|
|
446
|
+
};
|