ctxtop 0.0.0-stage → 4.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (262) hide show
  1. package/.antigravity-plugin/plugin.json +36 -0
  2. package/.claude-plugin/plugin.json +18 -0
  3. package/CHECKSUMS.sha256 +166 -0
  4. package/LICENSE +27 -0
  5. package/README.md +350 -2
  6. package/SECURITY.md +22 -0
  7. package/agents/ah-flash-scout.md +25 -0
  8. package/assets/advanced_view_loaded_420px.png +0 -0
  9. package/assets/hero-demo.svg +188 -0
  10. package/assets/logo.svg +19 -0
  11. package/assets/simple_view_lean_420px.png +0 -0
  12. package/assets/simple_view_loaded_420px.png +0 -0
  13. package/assets/social-preview.svg +72 -0
  14. package/assets/zone_c_step3_shield_full.png +0 -0
  15. package/bin/ah-statusline.mjs +175 -0
  16. package/bin/cli.mjs +3 -0
  17. package/bin/ctxtop.mjs +1799 -0
  18. package/commands/cost-radar.md +11 -0
  19. package/commands/evolve.md +10 -0
  20. package/commands/handoff.md +10 -0
  21. package/commands/horizon.md +10 -0
  22. package/commands/inspect.md +10 -0
  23. package/commands/precheck.md +10 -0
  24. package/config/default_flightcheck.json +133 -0
  25. package/config/modelCatalog.json +785 -0
  26. package/config/platformQuirks.json +19 -0
  27. package/hooks/handlers/compact-lifecycle.mjs +577 -0
  28. package/hooks/handlers/config-change.mjs +402 -0
  29. package/hooks/handlers/hookIo.mjs +311 -0
  30. package/hooks/handlers/hookIpcClient.mjs +83 -0
  31. package/hooks/handlers/post-model-switch.mjs +111 -0
  32. package/hooks/handlers/post-tool-use.mjs +366 -0
  33. package/hooks/handlers/pre-model-switch.mjs +1048 -0
  34. package/hooks/handlers/pre-tool-use.mjs +632 -0
  35. package/hooks/handlers/stop-gate.mjs +526 -0
  36. package/hooks/handlers/user-prompt-submit.mjs +367 -0
  37. package/hooks/hooks.json +131 -0
  38. package/hooks/post_invocation.mjs +13 -0
  39. package/hooks/post_tool_use.mjs +25 -0
  40. package/hooks/pre_invocation.mjs +14 -0
  41. package/hooks/pre_tool_use.mjs +97 -0
  42. package/hooks/register.js +726 -0
  43. package/hooks/session_start.mjs +24 -0
  44. package/hooks/stop.mjs +23 -0
  45. package/hooks.json +56 -0
  46. package/package.json +110 -4
  47. package/packages/antigravity/package.json +14 -0
  48. package/packages/antigravity/src/index.mjs +1 -0
  49. package/packages/claude-code/package.json +14 -0
  50. package/packages/claude-code/src/index.mjs +2 -0
  51. package/packages/core/package.json +14 -0
  52. package/packages/core/src/index.mjs +1 -0
  53. package/packages/engine-node/package.json +14 -0
  54. package/packages/engine-node/src/index.mjs +1 -0
  55. package/packages/mcp/package.json +17 -0
  56. package/packages/mcp/src/server.mjs +396 -0
  57. package/packages/pro/LICENSE-COMMERCIAL.md +19 -0
  58. package/packages/pro/README-SIGNING.md +30 -0
  59. package/packages/pro/package.json +16 -0
  60. package/packages/pro/src/index.mjs +3 -0
  61. package/packages/pro/src/team_finops_pack.mjs +241 -0
  62. package/packages/pro/workers/polar_webhook_worker.mjs +215 -0
  63. package/plugin.json +36 -0
  64. package/rules/AGENTS.md +10 -0
  65. package/sidecars/horizon-daemon/ipcServer.mjs +24 -0
  66. package/sidecars/horizon-daemon/server.mjs +2038 -0
  67. package/sidecars/horizon-daemon/sidecar.json +19 -0
  68. package/skills/cost-radar/SKILL.md +21 -0
  69. package/skills/evolve/SKILL.md +21 -0
  70. package/skills/handoff/SKILL.md +50 -0
  71. package/skills/horizon/SKILL.md +17 -0
  72. package/skills/inspect/SKILL.md +18 -0
  73. package/skills/precheck/SKILL.md +44 -0
  74. package/src/adapters/antigravity/index.mjs +109 -0
  75. package/src/adapters/claude-code/index.mjs +194 -0
  76. package/src/cli/cost_radar.mjs +70 -0
  77. package/src/cli/doctor.mjs +351 -0
  78. package/src/cli/horizon_hud.mjs +888 -0
  79. package/src/cli/init.mjs +425 -0
  80. package/src/cli/postmortem.mjs +515 -0
  81. package/src/coach/cacheGravityRouter.mjs +12 -0
  82. package/src/config/modelCatalog.json +704 -0
  83. package/src/config/platformQuirks.json +19 -0
  84. package/src/core/action_queue.mjs +264 -0
  85. package/src/core/ag_bridge.mjs +346 -0
  86. package/src/core/atomic_io.mjs +124 -0
  87. package/src/core/baseline.mjs +607 -0
  88. package/src/core/blindspot_sensors.mjs +399 -0
  89. package/src/core/bucket_b_runtime.mjs +1200 -0
  90. package/src/core/classifier.mjs +1757 -0
  91. package/src/core/coach/circuit_breakers.mjs +650 -0
  92. package/src/core/coach/contradiction_auditor.mjs +239 -0
  93. package/src/core/coach/effort_advisor.mjs +143 -0
  94. package/src/core/coach/handoff_hydrator.mjs +483 -0
  95. package/src/core/coach/progressive_checkpointer.mjs +551 -0
  96. package/src/core/coach/prompt_decomposer_coach.mjs +141 -0
  97. package/src/core/coach/reflector_queue.mjs +505 -0
  98. package/src/core/coach/saved_quota_ledger.mjs +615 -0
  99. package/src/core/coach/subagent_ipi_sanitizer.mjs +200 -0
  100. package/src/core/coach/subagent_phantom_auditor.mjs +261 -0
  101. package/src/core/coach/task_quota_router.mjs +955 -0
  102. package/src/core/coach/turn_zero_gate.mjs +574 -0
  103. package/src/core/compaction_coach.mjs +1126 -0
  104. package/src/core/config.mjs +214 -0
  105. package/src/core/constants.mjs +1987 -0
  106. package/src/core/detectors_det01_to_16.mjs +1523 -0
  107. package/src/core/doc_indexer.mjs +338 -0
  108. package/src/core/episodic_memory.mjs +135 -0
  109. package/src/core/fts5_output_compressor.mjs +359 -0
  110. package/src/core/git_ledger.mjs +189 -0
  111. package/src/core/handoff_audit.mjs +558 -0
  112. package/src/core/history.mjs +221 -0
  113. package/src/core/hook_diagnostics.mjs +158 -0
  114. package/src/core/horizon_classifier.mjs +326 -0
  115. package/src/core/horizon_config.mjs +802 -0
  116. package/src/core/host_watchdog.mjs +200 -0
  117. package/src/core/layer5_hygiene.mjs +1084 -0
  118. package/src/core/log_resolver.mjs +231 -0
  119. package/src/core/mcp_auditor.mjs +138 -0
  120. package/src/core/micro_skills_store.mjs +91 -0
  121. package/src/core/offenders.mjs +161 -0
  122. package/src/core/parser.mjs +1997 -0
  123. package/src/core/path_canonicalizer.mjs +73 -0
  124. package/src/core/paths.mjs +311 -0
  125. package/src/core/prefix_cache_shield.mjs +386 -0
  126. package/src/core/pro_license.mjs +385 -0
  127. package/src/core/prompt_rubric.mjs +518 -0
  128. package/src/core/pruner.mjs +433 -0
  129. package/src/core/quota.mjs +1860 -0
  130. package/src/core/rsi_instinct_engine.mjs +1197 -0
  131. package/src/core/rsl_clash_auditor.mjs +423 -0
  132. package/src/core/rsl_distiller.mjs +372 -0
  133. package/src/core/rsl_ledger.mjs +969 -0
  134. package/src/core/rsl_triage.mjs +909 -0
  135. package/src/core/sanitizer.mjs +121 -0
  136. package/src/core/session_auditor.mjs +658 -0
  137. package/src/core/session_forensics.mjs +303 -0
  138. package/src/core/session_timeline.mjs +358 -0
  139. package/src/core/settings.mjs +99 -0
  140. package/src/core/shared/detectors.mjs +1342 -0
  141. package/src/core/shared/fnv1a64.mjs +85 -0
  142. package/src/core/shared/handoff_schema.mjs +123 -0
  143. package/src/core/shared/index.mjs +5 -0
  144. package/src/core/shared/ipi_sanitizer.mjs +165 -0
  145. package/src/core/shared/micro_skills.mjs +1 -0
  146. package/src/core/shared/plugin_validator.mjs +202 -0
  147. package/src/core/shared/prompt_hygiene_governor.mjs +1 -0
  148. package/src/core/shield_manager.mjs +1417 -0
  149. package/src/core/sidecar_store.mjs +302 -0
  150. package/src/core/sprint_metrics.mjs +366 -0
  151. package/src/core/sqlite_client.mjs +283 -0
  152. package/src/core/storage_doctor.mjs +358 -0
  153. package/src/core/subagents.mjs +609 -0
  154. package/src/core/swarm_topology.mjs +360 -0
  155. package/src/core/task_quota_router.mjs +1656 -0
  156. package/src/core/telemetry/cache_ttl_watchdog.mjs +284 -0
  157. package/src/core/telemetry/scope_lock_registry.mjs +578 -0
  158. package/src/core/telemetry/shadow_file_detector.mjs +291 -0
  159. package/src/core/telemetry/trajectory_watcher.mjs +259 -0
  160. package/src/core/timeline_svg.mjs +236 -0
  161. package/src/core/tool_surgeon.mjs +305 -0
  162. package/src/core/trajectory_stitcher.mjs +645 -0
  163. package/src/core/u_shaped_optimizer.mjs +789 -0
  164. package/src/core/wal_hygiene.mjs +233 -0
  165. package/src/core-isomorphic/catalog/detectorCatalog.mjs +558 -0
  166. package/src/core-isomorphic/catalog/index.mjs +1 -0
  167. package/src/core-isomorphic/coach/cacheGravityRouter.mjs +1547 -0
  168. package/src/core-isomorphic/coach/microSkills.mjs +1 -0
  169. package/src/core-isomorphic/coach/promptHygieneGovernor.mjs +1 -0
  170. package/src/core-isomorphic/coach/repoCalibration.mjs +254 -0
  171. package/src/core-isomorphic/coach/resumeToolsetAuditor.mjs +321 -0
  172. package/src/core-isomorphic/coach/savedQuotaLedger.mjs +527 -0
  173. package/src/core-isomorphic/coach/smartCompactAdvisor.mjs +305 -0
  174. package/src/core-isomorphic/coach/stateBatonHydrator.mjs +810 -0
  175. package/src/core-isomorphic/coach/ttlWatchdog.mjs +464 -0
  176. package/src/core-isomorphic/config/anthropicModelCatalog.json +606 -0
  177. package/src/core-isomorphic/config/catalogLoader.mjs +1268 -0
  178. package/src/core-isomorphic/config/horizonConfig.mjs +1055 -0
  179. package/src/core-isomorphic/control/callTree.mjs +338 -0
  180. package/src/core-isomorphic/control/coachLatches.mjs +126 -0
  181. package/src/core-isomorphic/control/controlStateMachine.mjs +205 -0
  182. package/src/core-isomorphic/control/index.mjs +49 -0
  183. package/src/core-isomorphic/control/swarmWorktree.mjs +109 -0
  184. package/src/core-isomorphic/control/toolSuspension.mjs +197 -0
  185. package/src/core-isomorphic/diagnostics/drift_guard.mjs +382 -0
  186. package/src/core-isomorphic/diagnostics/index.mjs +9 -0
  187. package/src/core-isomorphic/diagnostics/micro_skills.mjs +471 -0
  188. package/src/core-isomorphic/diagnostics/prompt_hygiene_governor.mjs +888 -0
  189. package/src/core-isomorphic/economics/billingBlocks.mjs +251 -0
  190. package/src/core-isomorphic/economics/cacheGravity.mjs +369 -0
  191. package/src/core-isomorphic/economics/cliffBreaker.mjs +256 -0
  192. package/src/core-isomorphic/economics/driftLatency.mjs +433 -0
  193. package/src/core-isomorphic/economics/dualTtl.mjs +198 -0
  194. package/src/core-isomorphic/economics/index.mjs +47 -0
  195. package/src/core-isomorphic/economics/retentionRadar.mjs +348 -0
  196. package/src/core-isomorphic/guards/bucketBAlphaBeta.mjs +860 -0
  197. package/src/core-isomorphic/guards/bucketBGamma.mjs +442 -0
  198. package/src/core-isomorphic/guards/bucketBHeuristics.mjs +37 -0
  199. package/src/core-isomorphic/guards/circuitBreakers.mjs +1202 -0
  200. package/src/core-isomorphic/guards/det01_03_shell_secret.mjs +1184 -0
  201. package/src/core-isomorphic/guards/det04_12_thrash_rot.mjs +572 -0
  202. package/src/core-isomorphic/guards/det13_20_mcp_scope.mjs +913 -0
  203. package/src/core-isomorphic/guards/docIndexCore.mjs +446 -0
  204. package/src/core-isomorphic/guards/losslessPostSanitizer.mjs +317 -0
  205. package/src/core-isomorphic/guards/slopsquatGate.mjs +551 -0
  206. package/src/core-isomorphic/index.js +27 -0
  207. package/src/core-isomorphic/telemetry/otelExporter.mjs +187 -0
  208. package/src/core-isomorphic/telemetry/scopeLockRegistry.mjs +522 -0
  209. package/src/core-isomorphic/telemetry/sprintMetrics.mjs +327 -0
  210. package/src/core-isomorphic/telemetry/swarmTopology.mjs +152 -0
  211. package/src/core-isomorphic/telemetry/telemetryBridge.mjs +1156 -0
  212. package/src/core-isomorphic/ui/brailleSparkline.mjs +142 -0
  213. package/src/core-isomorphic/ui/costRadarSvg.mjs +276 -0
  214. package/src/core-isomorphic/ui/horizonGrid.mjs +475 -0
  215. package/src/core-isomorphic/ui/horizonPanelHtml.mjs +1237 -0
  216. package/src/core-isomorphic/ui/modViews.mjs +1504 -0
  217. package/src/core-isomorphic/ui/pdfReceiptGenerator.mjs +329 -0
  218. package/src/core-isomorphic/ustf/index.mjs +7 -0
  219. package/src/core-isomorphic/ustf/normalizer.mjs +512 -0
  220. package/src/core-isomorphic/ustf/schema.mjs +186 -0
  221. package/src/core-isomorphic/utils/runtime.mjs +962 -0
  222. package/src/demo/synthetic_states.mjs +661 -0
  223. package/src/diagnostics/bugReportGenerator.mjs +253 -0
  224. package/src/diagnostics/claudeHookCapabilities.mjs +396 -0
  225. package/src/diagnostics/doctorEngine.mjs +295 -0
  226. package/src/engine-node/db/index.mjs +58 -0
  227. package/src/engine-node/db/jsonl_fallback.mjs +184 -0
  228. package/src/engine-node/db/probe.mjs +342 -0
  229. package/src/engine-node/db/sqlite_wrapper.mjs +122 -0
  230. package/src/engine-node/index.mjs +12 -0
  231. package/src/engine-node/ipc/client.mjs +447 -0
  232. package/src/engine-node/ipc/index.mjs +22 -0
  233. package/src/engine-node/ipc/protocol.mjs +177 -0
  234. package/src/engine-node/ipc/server.mjs +445 -0
  235. package/src/engine-node/offloader/index.mjs +6 -0
  236. package/src/engine-node/offloader/tool_offloader.mjs +241 -0
  237. package/src/engine-node/sensors/blindspots.mjs +376 -0
  238. package/src/engine-node/sensors/git_merge_tree.mjs +223 -0
  239. package/src/engine-node/sensors/index.mjs +7 -0
  240. package/src/hooks/antigravity_rsl_hook.mjs +250 -0
  241. package/src/hooks/claude_rsl_hook.mjs +288 -0
  242. package/src/hooks/portfolio_runtime.mjs +858 -0
  243. package/src/hooks/post_invocation.mjs +611 -0
  244. package/src/hooks/pre_invocation.mjs +1725 -0
  245. package/src/hooks/session_start.mjs +199 -0
  246. package/src/shims/agy_cli_compressor.mjs +705 -0
  247. package/src/shims/agy_git_guard.mjs +749 -0
  248. package/src/shims/ah_log_query.mjs +39 -0
  249. package/src/ui/costRadarSvg.mjs +6 -0
  250. package/src/ui/horizonGrid.mjs +1 -0
  251. package/src/ui/horizonPanelHtml.mjs +1 -0
  252. package/src/ui/horizon_panel.css +167 -0
  253. package/src/ui/horizon_panel.js +212 -0
  254. package/src/ui/modViews.mjs +1 -0
  255. package/src/utils/modelSwitchTelemetry.mjs +308 -0
  256. package/src/utils/nodeIo.mjs +59 -0
  257. package/src/utils/runtime.mjs +1 -0
  258. package/src/utils/sessionCalibration.mjs +657 -0
  259. package/src/webview/advanced_view.html +1808 -0
  260. package/src/webview/horizon_panel.html +12397 -0
  261. package/src/webview/registry.mjs +80 -0
  262. package/src/webview/simple_view.html +1370 -0
@@ -0,0 +1,446 @@
1
+ /**
2
+ * @file src/guards/docIndexCore.mjs
3
+ * Pure JavaScript zero-token structural document & log indexer (`< 20ms`, 0 LLM tokens).
4
+ *
5
+ * Architectural Invariants:
6
+ * - ZERO `node:*` imports, ZERO `require()`, ZERO dynamic ESM imports (Mod Law 4 compliant).
7
+ * - Strictly excludes `ctxvault` SKU paths (`packages/airlock/`, `airlock-schema-canary.yml`,
8
+ * `docs/HIGHBEAM_AIRLOCK_*.md`).
9
+ * - Scans outside fenced code blocks for Markdown headings (`#`..`####`), code definitions
10
+ * (`export`, `function`, `class`, `interface`, `type`, `def`, `async def`, `func`, `struct`),
11
+ * or top-level JSON/YAML keys, with a contiguous 150-line unstructured fallback when `< 3`
12
+ * structural boundaries exist in files `> 400` lines.
13
+ */
14
+
15
+ import { normalizeSlashes, sha256Hex } from '../utils/runtime.mjs';
16
+
17
+ export const UNSTRUCTURED_FALLBACK_LINE_THRESHOLD = 400;
18
+ export const UNSTRUCTURED_SPAN_SIZE = 150;
19
+ export const MIN_STRUCTURAL_BOUNDARIES = 3;
20
+ export const MAX_IDENTIFIERS_PER_SPAN = 8;
21
+
22
+ const STOPWORDS = new Set([
23
+ 'const', 'let', 'var', 'function', 'async', 'await', 'return', 'export', 'import', 'from',
24
+ 'default', 'class', 'interface', 'type', 'extends', 'implements', 'public', 'private',
25
+ 'protected', 'static', 'true', 'false', 'null', 'undefined', 'this', 'super', 'new',
26
+ 'throw', 'catch', 'try', 'finally', 'else', 'switch', 'case', 'break', 'continue',
27
+ 'while', 'for', 'typeof', 'instanceof', 'void', 'delete', 'with', 'yield', 'def',
28
+ 'self', 'none', 'pass', 'elif', 'lambda', 'func', 'struct', 'impl', 'pub', 'mut',
29
+ 'string', 'number', 'boolean', 'object', 'array', 'promise', 'record', 'the', 'and',
30
+ 'with', 'that', 'this', 'from', 'into', 'when', 'where', 'then', 'only', 'also',
31
+ 'must', 'should', 'never', 'always', 'line', 'lines', 'file', 'files', 'test', 'data'
32
+ ]);
33
+
34
+ /**
35
+ * Checks whether a path belongs to the isolated sister security SKU and must never be indexed or read.
36
+ *
37
+ * @param {string} [filePath='']
38
+ * @returns {boolean}
39
+ */
40
+ export function isForbiddenSkuPath(filePath = '') {
41
+ const norm = normalizeSlashes(String(filePath || '')).trim();
42
+ if (!norm) return false;
43
+ const lower = norm.toLowerCase();
44
+ if (
45
+ /(?:^|\/)packages\/airlock(?:\/|$)/i.test(lower) ||
46
+ lower.endsWith('.github/workflows/airlock-schema-canary.yml') ||
47
+ /(?:^|\/)\.github\/workflows\/airlock-schema-canary\.yml$/i.test(lower) ||
48
+ /(?:^|\/)docs\/highbeam_airlock_[^/]*\.md$/i.test(lower)
49
+ ) {
50
+ return true;
51
+ }
52
+ return false;
53
+ }
54
+
55
+ /**
56
+ * Splits text into lines across CRLF and LF line endings.
57
+ *
58
+ * @param {string} [rawText='']
59
+ * @returns {string[]}
60
+ */
61
+ export function splitLinesNormalized(rawText = '') {
62
+ const str = String(rawText ?? '');
63
+ if (str.length === 0) return [];
64
+ return str.replace(/\r\n/g, '\n').replace(/\r/g, '\n').split('\n');
65
+ }
66
+
67
+ /**
68
+ * Parses YAML frontmatter from a `.index.md` document.
69
+ *
70
+ * @param {string} [markdownText='']
71
+ * @returns {{ indexed_file: string, sha256: string, total_lines: number, generated_by: string } | null}
72
+ */
73
+ export function parseIndexFrontmatter(markdownText = '') {
74
+ const lines = splitLinesNormalized(markdownText);
75
+ if (lines.length < 3 || lines[0].trim() !== '---') return null;
76
+ const meta = {
77
+ indexed_file: '',
78
+ sha256: '',
79
+ total_lines: 0,
80
+ generated_by: ''
81
+ };
82
+ let closed = false;
83
+ for (let i = 1; i < Math.min(lines.length, 25); i++) {
84
+ const trimmed = lines[i].trim();
85
+ if (trimmed === '---') {
86
+ closed = true;
87
+ break;
88
+ }
89
+ const sepIdx = trimmed.indexOf(':');
90
+ if (sepIdx === -1) continue;
91
+ const key = trimmed.slice(0, sepIdx).trim();
92
+ const val = trimmed.slice(sepIdx + 1).trim().replace(/^["']|["']$/g, '');
93
+ if (key === 'indexed_file') meta.indexed_file = val;
94
+ else if (key === 'sha256') meta.sha256 = val;
95
+ else if (key === 'total_lines') meta.total_lines = Number(val) || 0;
96
+ else if (key === 'generated_by') meta.generated_by = val;
97
+ }
98
+ return closed && meta.sha256 ? meta : null;
99
+ }
100
+
101
+ /**
102
+ * Compares live SHA-256 of a file against `.index.md` YAML frontmatter.
103
+ * Returns `0` (`FRESH`), `1` (`STALE`), or `2` (`MISSING`).
104
+ *
105
+ * @param {string} liveSha256
106
+ * @param {string | null | undefined} indexMarkdownText
107
+ * @returns {{ code: 0 | 1 | 2, status: 'FRESH' | 'STALE' | 'MISSING', recordedSha256: string | null }}
108
+ */
109
+ export function compareIndexFreshness(liveSha256, indexMarkdownText) {
110
+ if (!indexMarkdownText || typeof indexMarkdownText !== 'string') {
111
+ return { code: 2, status: 'MISSING', recordedSha256: null };
112
+ }
113
+ const fm = parseIndexFrontmatter(indexMarkdownText);
114
+ if (!fm || !fm.sha256) {
115
+ return { code: 2, status: 'MISSING', recordedSha256: null };
116
+ }
117
+ if (String(fm.sha256).toLowerCase() === String(liveSha256 || '').toLowerCase()) {
118
+ return { code: 0, status: 'FRESH', recordedSha256: fm.sha256 };
119
+ }
120
+ return { code: 1, status: 'STALE', recordedSha256: fm.sha256 };
121
+ }
122
+
123
+ function sanitizeTableCell(text = '') {
124
+ return String(text || '')
125
+ .replace(/\|/g, '\\|')
126
+ .replace(/\s+/g, ' ')
127
+ .trim()
128
+ .slice(0, 96);
129
+ }
130
+
131
+ /**
132
+ * Extracts up to `maxCount` backticked terms or high-signal identifiers from a slice of lines.
133
+ *
134
+ * @param {string[]} lines
135
+ * @param {number} startIdx - 0-based inclusive
136
+ * @param {number} endIdx - 0-based inclusive
137
+ * @param {number} [maxCount=MAX_IDENTIFIERS_PER_SPAN]
138
+ * @returns {string[]}
139
+ */
140
+ export function extractSpanIdentifiers(lines, startIdx, endIdx, maxCount = MAX_IDENTIFIERS_PER_SPAN) {
141
+ const counts = new Map();
142
+ const backtickTerms = [];
143
+ const backtickSeen = new Set();
144
+
145
+ const upper = Math.min(lines.length - 1, endIdx);
146
+ for (let i = Math.max(0, startIdx); i <= upper; i++) {
147
+ const line = lines[i] || '';
148
+ if (line.length > 500) continue;
149
+
150
+ // 1. Backticked terms first (high signal in Markdown/JSDoc/logs)
151
+ const btRegex = /`([A-Za-z0-9_./:-]{3,48})`/g;
152
+ let btMatch;
153
+ while ((btMatch = btRegex.exec(line)) !== null) {
154
+ const term = btMatch[1];
155
+ if (!backtickSeen.has(term) && !STOPWORDS.has(term.toLowerCase())) {
156
+ backtickSeen.add(term);
157
+ backtickTerms.push(term);
158
+ if (backtickTerms.length >= maxCount) break;
159
+ }
160
+ }
161
+
162
+ // 2. Identifiers (camelCase, PascalCase, snake_case, SCREAMING_SNAKE, error codes)
163
+ const idRegex = /\b([A-Za-z_][A-Za-z0-9_]{3,44})\b/g;
164
+ let idMatch;
165
+ while ((idMatch = idRegex.exec(line)) !== null) {
166
+ const ident = idMatch[1];
167
+ const lower = ident.toLowerCase();
168
+ if (STOPWORDS.has(lower)) continue;
169
+ const isStructuredIdent =
170
+ /[A-Z]/.test(ident) || ident.includes('_') || ident.length >= 6;
171
+ if (!isStructuredIdent) continue;
172
+ counts.set(ident, (counts.get(ident) || 0) + 1);
173
+ }
174
+ }
175
+
176
+ const out = [...backtickTerms.slice(0, maxCount)];
177
+ if (out.length < maxCount) {
178
+ const ranked = Array.from(counts.entries())
179
+ .filter(([ident]) => !backtickSeen.has(ident))
180
+ .sort((a, b) => b[1] - a[1] || a[0].localeCompare(b[0]));
181
+ for (const [ident] of ranked) {
182
+ out.push(ident);
183
+ if (out.length >= maxCount) break;
184
+ }
185
+ }
186
+ return out;
187
+ }
188
+
189
+ /**
190
+ * Detects structural boundary label on a single line (outside Markdown code fences).
191
+ *
192
+ * @param {string} line
193
+ * @param {boolean} isMarkdownLike
194
+ * @returns {string | null}
195
+ */
196
+ function matchStructuralBoundary(line, isMarkdownLike) {
197
+ const trimmed = line.trim();
198
+ if (!trimmed || trimmed.length > 240) return null;
199
+
200
+ // Markdown headings (# .. ####)
201
+ const mdMatch = /^(#{1,4})\s+(.+)$/.exec(line);
202
+ if (mdMatch) {
203
+ return `${mdMatch[1]} ${mdMatch[2].trim().slice(0, 72)}`;
204
+ }
205
+ if (isMarkdownLike) return null;
206
+
207
+ // JS/TS exports, functions, classes, interfaces, types
208
+ const jsMatch =
209
+ /^(?:export\s+(?:default\s+)?)?(?:async\s+)?(?:function\*?\s+([A-Za-z_$][A-Za-z0-9_$]*)|class\s+([A-Za-z_$][A-Za-z0-9_$]*)|interface\s+([A-Za-z_$][A-Za-z0-9_$]*)|type\s+([A-Za-z_$][A-Za-z0-9_$]*)\s*=|const\s+([A-Za-z_$][A-Za-z0-9_$]*)\s*=\s*(?:async\s*)?(?:\([^\x29]*\)|[A-Za-z_$][A-Za-z0-9_$]*)\s*=>|(?:export\s+const\s+([A-Za-z_$][A-Za-z0-9_$]*)\s*=))/.exec(
210
+ line
211
+ );
212
+ if (jsMatch) {
213
+ const sym = jsMatch[1] || jsMatch[2] || jsMatch[3] || jsMatch[4] || jsMatch[5] || jsMatch[6];
214
+ if (sym) return sym;
215
+ }
216
+
217
+ // Python / Go / Rust definitions
218
+ const pyGoRustMatch =
219
+ /^(?:async\s+def|def|class|func(?:\s*\([^\x29]+\))?|pub\s+(?:async\s+)?fn|fn|pub\s+struct|struct|impl)\s+([A-Za-z_][A-Za-z0-9_]*)/.exec(
220
+ line
221
+ );
222
+ if (pyGoRustMatch) {
223
+ return pyGoRustMatch[1];
224
+ }
225
+
226
+ // Top-level JSON or YAML keys (0-2 leading spaces)
227
+ const jsonKeyMatch = /^ {0,2}"([A-Za-z0-9_.-]{2,48})"\s*:\s*[\x7b\x5b]/.exec(line);
228
+ if (jsonKeyMatch) {
229
+ return `"${jsonKeyMatch[1]}"`;
230
+ }
231
+ const yamlKeyMatch = /^([A-Za-z][A-Za-z0-9_.-]{1,48}):\s*(?:#.*)?$/.exec(line);
232
+ if (yamlKeyMatch && !/^(?:https?|file|mailto)$/i.test(yamlKeyMatch[1])) {
233
+ return `${yamlKeyMatch[1]}:`;
234
+ }
235
+
236
+ return null;
237
+ }
238
+
239
+ /**
240
+ * Builds deterministic zero-token `.index.md` content for any text/code/log file.
241
+ *
242
+ * @param {object} params
243
+ * @param {string} params.filePath
244
+ * @param {string} [params.displayPath]
245
+ * @param {string} params.rawText
246
+ * @param {string} [params.sha256]
247
+ * @returns {{
248
+ * markdown: string,
249
+ * sha256: string,
250
+ * totalLines: number,
251
+ * spanCount: number,
252
+ * usedFallback: boolean,
253
+ * indexTokens: number
254
+ * }}
255
+ */
256
+ export function buildStructuralIndexMarkdown({
257
+ filePath = '',
258
+ displayPath = '',
259
+ rawText = '',
260
+ sha256 = ''
261
+ } = {}) {
262
+ const normalizedRaw = String(rawText ?? '').replace(/\r\n/g, '\n');
263
+ const computedSha = sha256 || sha256Hex(normalizedRaw);
264
+ const lines = splitLinesNormalized(rawText);
265
+ const totalLines = lines.length;
266
+ const labelPath = normalizeSlashes(displayPath || filePath || 'unknown');
267
+ const isMarkdownLike = /\.(?:md|mdx|markdown)$/i.test(labelPath);
268
+
269
+ const boundaries = [];
270
+ let inFence = false;
271
+ let fenceMarker = '';
272
+
273
+ for (let i = 0; i < totalLines; i++) {
274
+ const line = lines[i];
275
+ const fenceMatch = /^\s*(`{3,}|~{3,})/.exec(line);
276
+ if (fenceMatch) {
277
+ const marker = fenceMatch[1][0];
278
+ if (!inFence) {
279
+ inFence = true;
280
+ fenceMarker = marker;
281
+ } else if (marker === fenceMarker) {
282
+ inFence = false;
283
+ fenceMarker = '';
284
+ }
285
+ continue;
286
+ }
287
+ if (inFence) continue;
288
+
289
+ const label = matchStructuralBoundary(line, isMarkdownLike);
290
+ if (label) {
291
+ boundaries.push({ lineNum: i + 1, label });
292
+ }
293
+ }
294
+
295
+ let spans = [];
296
+ let usedFallback = false;
297
+
298
+ if (boundaries.length < MIN_STRUCTURAL_BOUNDARIES && totalLines > UNSTRUCTURED_FALLBACK_LINE_THRESHOLD) {
299
+ usedFallback = true;
300
+ for (let start = 1; start <= totalLines; start += UNSTRUCTURED_SPAN_SIZE) {
301
+ const end = Math.min(totalLines, start + UNSTRUCTURED_SPAN_SIZE - 1);
302
+ const idents = extractSpanIdentifiers(lines, start - 1, end - 1);
303
+ spans.push({
304
+ startLine: start,
305
+ endLine: end,
306
+ label: `Unstructured Span L${start}-L${end}`,
307
+ identifiers: idents
308
+ });
309
+ }
310
+ } else if (boundaries.length > 0) {
311
+ // Include leading preamble if boundary 0 starts after line 1
312
+ if (boundaries[0].lineNum > 1) {
313
+ const preEnd = boundaries[0].lineNum - 1;
314
+ spans.push({
315
+ startLine: 1,
316
+ endLine: preEnd,
317
+ label: 'Preamble / Imports',
318
+ identifiers: extractSpanIdentifiers(lines, 0, preEnd - 1)
319
+ });
320
+ }
321
+ for (let idx = 0; idx < boundaries.length; idx++) {
322
+ const curr = boundaries[idx];
323
+ const next = boundaries[idx + 1];
324
+ const endLine = next ? Math.max(curr.lineNum, next.lineNum - 1) : totalLines;
325
+ spans.push({
326
+ startLine: curr.lineNum,
327
+ endLine,
328
+ label: curr.label,
329
+ identifiers: extractSpanIdentifiers(lines, curr.lineNum - 1, endLine - 1)
330
+ });
331
+ }
332
+ } else {
333
+ const end = Math.max(1, totalLines);
334
+ spans.push({
335
+ startLine: 1,
336
+ endLine: end,
337
+ label: totalLines === 0 ? 'Empty File' : `Full Span L1-L${end}`,
338
+ identifiers: totalLines === 0 ? [] : extractSpanIdentifiers(lines, 0, end - 1)
339
+ });
340
+ }
341
+
342
+ // Cap extreme table rows at 120 spans by coalescing adjacent micro-spans if needed
343
+ if (spans.length > 120) {
344
+ const bucketSize = Math.ceil(spans.length / 80);
345
+ const merged = [];
346
+ for (let i = 0; i < spans.length; i += bucketSize) {
347
+ const group = spans.slice(i, i + bucketSize);
348
+ const combinedIdents = [];
349
+ const seen = new Set();
350
+ for (const g of group) {
351
+ for (const id of g.identifiers) {
352
+ if (!seen.has(id)) {
353
+ seen.add(id);
354
+ combinedIdents.push(id);
355
+ }
356
+ if (combinedIdents.length >= MAX_IDENTIFIERS_PER_SPAN) break;
357
+ }
358
+ }
359
+ merged.push({
360
+ startLine: group[0].startLine,
361
+ endLine: group[group.length - 1].endLine,
362
+ label: group.length > 1 ? `${group[0].label} (+${group.length - 1} more)` : group[0].label,
363
+ identifiers: combinedIdents
364
+ });
365
+ }
366
+ spans = merged;
367
+ }
368
+
369
+ const rows = spans.map((sp) => {
370
+ const rangeStr = `L${sp.startLine}-L${sp.endLine}`;
371
+ const sectionStr = sanitizeTableCell(sp.label);
372
+ const identStr =
373
+ sp.identifiers.length > 0
374
+ ? sp.identifiers.map((id) => `\`${sanitizeTableCell(id)}\``).join(', ')
375
+ : '—';
376
+ return `| ${rangeStr} | ${sectionStr} | ${identStr} |`;
377
+ });
378
+
379
+ const markdown = [
380
+ '---',
381
+ `indexed_file: "${labelPath}"`,
382
+ `sha256: "${computedSha}"`,
383
+ `total_lines: ${totalLines}`,
384
+ 'generated_by: "ctxtop-doc-indexer"',
385
+ '---',
386
+ '',
387
+ `# Structural Index: \`${labelPath}\` (${totalLines} lines)`,
388
+ '',
389
+ '| Lines | Section / Symbol | High-Signal Identifiers |',
390
+ '| :--- | :--- | :--- |',
391
+ ...rows,
392
+ ''
393
+ ].join('\n');
394
+
395
+ return {
396
+ markdown,
397
+ sha256: computedSha,
398
+ totalLines,
399
+ spanCount: spans.length,
400
+ usedFallback,
401
+ indexTokens: Math.max(1, Math.ceil(markdown.length / 4))
402
+ };
403
+ }
404
+
405
+ /**
406
+ * Formats 1-indexed line-numbered slice output (` 142 | ...`) across CRLF and LF inputs.
407
+ *
408
+ * @param {string} [rawText='']
409
+ * @param {number} [startLine=1]
410
+ * @param {number} [endLine=1]
411
+ * @returns {{ text: string, startLine: number, endLine: number, totalLines: number }}
412
+ */
413
+ export function formatSlicedLines(rawText = '', startLine = 1, endLine = 1) {
414
+ const lines = splitLinesNormalized(rawText);
415
+ const totalLines = lines.length;
416
+ if (totalLines === 0) {
417
+ return { text: '', startLine: 0, endLine: 0, totalLines: 0 };
418
+ }
419
+ const s = Math.max(1, Math.min(totalLines, Math.floor(Number(startLine) || 1)));
420
+ const e = Math.max(s, Math.min(totalLines, Math.floor(Number(endLine) || s)));
421
+ const width = Math.max(6, String(e).length);
422
+ const out = [];
423
+ for (let lineNum = s; lineNum <= e; lineNum++) {
424
+ out.push(`${String(lineNum).padStart(width, ' ')} | ${lines[lineNum - 1]}`);
425
+ }
426
+ return {
427
+ text: out.join('\n'),
428
+ startLine: s,
429
+ endLine: e,
430
+ totalLines
431
+ };
432
+ }
433
+
434
+ export default {
435
+ UNSTRUCTURED_FALLBACK_LINE_THRESHOLD,
436
+ UNSTRUCTURED_SPAN_SIZE,
437
+ MIN_STRUCTURAL_BOUNDARIES,
438
+ MAX_IDENTIFIERS_PER_SPAN,
439
+ isForbiddenSkuPath,
440
+ splitLinesNormalized,
441
+ parseIndexFrontmatter,
442
+ compareIndexFreshness,
443
+ extractSpanIdentifiers,
444
+ buildStructuralIndexMarkdown,
445
+ formatSlicedLines
446
+ };