@cassiomc1/forgeloop 1.10.2 → 1.11.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (84) hide show
  1. package/.cursor/rules/project-loop.mdc +2 -2
  2. package/.forgeloop/forgeloop.gitignore +1 -0
  3. package/.github/copilot-instructions.md +2 -2
  4. package/AGENTS.md +1 -0
  5. package/AGENT_COMPATIBILITY.md +7 -0
  6. package/CLAUDE.md +1 -0
  7. package/DOCS_INDEX.md +10 -0
  8. package/LOOP_SYSTEM_DESIGN.md +12 -0
  9. package/ORCHESTRATOR_INTEGRATION.md +9 -0
  10. package/PROTOCOL_INTEGRATION.md +17 -0
  11. package/README.md +12 -10
  12. package/THIRD_PARTY_NOTICES.md +11 -0
  13. package/THREAT_MODEL.md +21 -0
  14. package/benchmarks/repository-index/README.md +73 -0
  15. package/benchmarks/repository-index/queries.json +12 -0
  16. package/benchmarks/repository-index/run-hot-path.mjs +142 -0
  17. package/benchmarks/repository-index/run-persistent-transport.mjs +169 -0
  18. package/completions/_forgeloop +7 -1
  19. package/completions/forgeloop.bash +13 -1
  20. package/completions/forgeloop.fish +40 -1
  21. package/docs/AGENT_PROTOCOL_SUMMARY.md +8 -1
  22. package/docs/CLI_REFERENCE.md +114 -2
  23. package/docs/DOCUMENTATION_GUIDE.md +10 -5
  24. package/docs/GETTING_STARTED.md +17 -0
  25. package/docs/MCP.md +16 -1
  26. package/docs/PACKAGE_CONTENTS.md +7 -1
  27. package/docs/PERSISTENT_SEARCH_TRANSPORT.md +289 -0
  28. package/docs/RECIPES.md +30 -0
  29. package/docs/RELEASE_CHECKLIST.md +34 -0
  30. package/docs/REPOSITORY_INDEX.md +553 -0
  31. package/docs/TROUBLESHOOTING.md +147 -0
  32. package/docs/UNIVERSAL_INTEGRATION.md +30 -0
  33. package/docs/diagrams/README.md +10 -0
  34. package/package.json +11 -2
  35. package/scripts/update-tgrep-manifest.mjs +86 -0
  36. package/scripts/verify-tgrep-manifest.mjs +17 -0
  37. package/src/cli.js +35 -0
  38. package/src/commands/doctor.js +76 -1
  39. package/src/commands/index-rebuild.js +1 -0
  40. package/src/commands/index-setup.js +1 -0
  41. package/src/commands/index-start.js +1 -0
  42. package/src/commands/index-status.js +1 -0
  43. package/src/commands/index-stop.js +1 -0
  44. package/src/commands/init.js +39 -1
  45. package/src/commands/repository-index.js +111 -0
  46. package/src/commands/search.js +1 -0
  47. package/src/commands/update.js +32 -4
  48. package/src/core/cli-command-definitions.js +97 -3
  49. package/src/core/command-executors.js +35 -2
  50. package/src/core/command-input.js +23 -0
  51. package/src/core/error-codes.js +195 -0
  52. package/src/core/filesystem.js +10 -1
  53. package/src/core/integration-invocation-policy.js +27 -0
  54. package/src/core/integration-resources.js +16 -1
  55. package/src/core/protocol-info.js +23 -0
  56. package/src/integration.d.ts +90 -0
  57. package/src/integration.js +21 -0
  58. package/src/persistent-transport/client.js +293 -0
  59. package/src/persistent-transport/constants.js +24 -0
  60. package/src/persistent-transport/errors.js +38 -0
  61. package/src/persistent-transport/framing.js +61 -0
  62. package/src/persistent-transport/lifecycle.js +116 -0
  63. package/src/persistent-transport/ownership.js +184 -0
  64. package/src/persistent-transport/paths.js +31 -0
  65. package/src/persistent-transport/protocol.js +95 -0
  66. package/src/persistent-transport/server.js +256 -0
  67. package/src/persistent-transport/state.js +49 -0
  68. package/src/repository-index/args.js +59 -0
  69. package/src/repository-index/binary-manager.js +413 -0
  70. package/src/repository-index/constants.js +45 -0
  71. package/src/repository-index/errors.js +38 -0
  72. package/src/repository-index/lifecycle.js +17 -0
  73. package/src/repository-index/lock.js +113 -0
  74. package/src/repository-index/manifest.js +132 -0
  75. package/src/repository-index/metrics.js +30 -0
  76. package/src/repository-index/normalize-json.js +187 -0
  77. package/src/repository-index/paths.js +39 -0
  78. package/src/repository-index/platform.js +20 -0
  79. package/src/repository-index/process.js +140 -0
  80. package/src/repository-index/readiness.js +62 -0
  81. package/src/repository-index/search.js +262 -0
  82. package/src/repository-index/server.js +432 -0
  83. package/src/repository-index/status.js +397 -0
  84. package/src/repository-index/tgrep-manifest.json +38 -0
@@ -33,6 +33,10 @@ This guide provides symptom-first recovery procedures for common ForgeLoop proto
33
33
  - [Structural-quality verification reports a regression](#symptom-structural-quality-verification-reports-a-regression)
34
34
  - [Structural-quality provider is unavailable or invalid](#symptom-structural-quality-provider-is-unavailable-or-invalid)
35
35
  - [Structural-quality evidence is stale or incomparable](#symptom-structural-quality-evidence-is-stale-or-incomparable)
36
+ - [Repository Index is not ready](#symptom-repository-index-is-not-ready)
37
+ - [Repository search fails](#symptom-repository-search-fails)
38
+ - [Persistent search host is unavailable](#symptom-persistent-search-host-is-unavailable)
39
+ - [Persistent search host ownership is unverified or stale](#symptom-persistent-search-host-ownership-is-unverified-or-stale)
36
40
  - [Another harness cannot resume the task](#symptom-another-harness-cannot-resume)
37
41
  - [Task claim conflict or recovered task](#symptom-task-creation-blocked-by-a-write-claim-conflict-e_task_scope_conflict)
38
42
  - [Stable Error & Reason Code Reference](#stable-error-and-reason-codes)
@@ -1016,6 +1020,120 @@ Resolve the legitimate drift through the normal task lifecycle. Do not edit a
1016
1020
  baseline or evaluation by hand, and do not use a bundle as a reason to bypass
1017
1021
  current-cycle validation.
1018
1022
 
1023
+ ### Symptom: Repository Index is not ready
1024
+
1025
+ #### What it means
1026
+
1027
+ `forgeloop index-status` did not report `READY`. For a Git repository the
1028
+ engine is mandatory, so `doctor` and repository-wide search remain unhealthy
1029
+ until the pinned binary, derived index, and ForgeLoop-owned watcher are
1030
+ verified.
1031
+
1032
+ #### Inspect
1033
+
1034
+ ```bash
1035
+ forgeloop index-status --json
1036
+ forgeloop doctor --json
1037
+ ```
1038
+
1039
+ #### Safe recovery
1040
+
1041
+ 1. Run `forgeloop index-start --json` when the index is complete but the
1042
+ watcher is down.
1043
+ 2. Run `forgeloop index-setup --asset /absolute/path/to/pinned-asset.tar.gz`
1044
+ for a missing engine or an air-gapped host.
1045
+ 3. Run `forgeloop index-rebuild --json` after corruption or a confirmed stale
1046
+ index.
1047
+
1048
+ Never delete the whole `.forgeloop` directory and never stop a process by
1049
+ executable name. A status of `DEFERRED` for a non-Git protocol fixture is
1050
+ intentional and is not evidence that a real repository is healthy.
1051
+
1052
+ ### Symptom: Repository search fails
1053
+
1054
+ #### What it means
1055
+
1056
+ The provider-neutral search service rejected the request, could not verify the
1057
+ managed engine, or received an invalid native result. Search never falls back
1058
+ silently to `rg`, `grep`, or an arbitrary `PATH` executable.
1059
+
1060
+ #### Inspect
1061
+
1062
+ ```bash
1063
+ forgeloop search "pattern" --json
1064
+ forgeloop index-status --json
1065
+ ```
1066
+
1067
+ Correct request bounds such as the pattern, filters, context, or `max-count`
1068
+ when the error is `E_REPOSITORY_INDEX_REQUEST_INVALID`. For
1069
+ `E_REPOSITORY_INDEX_OUTPUT_INVALID`, preserve the bounded diagnostic and
1070
+ rebuild; repeated malformed output indicates a pinned-engine regression.
1071
+ Native exit code `1` means no matches and is normalized to a successful empty
1072
+ ForgeLoop result. Native exit code `2` is an actual search failure.
1073
+
1074
+ ### Symptom: Persistent search host is unavailable
1075
+
1076
+ #### What it means
1077
+
1078
+ The CLI-only local transport could not connect to or start its user-scoped
1079
+ ForgeLoop host within the bounded startup/request timeout. This is a transport
1080
+ failure, not permission to bypass the mandatory Repository Index or switch to
1081
+ `rg`, `grep`, or an arbitrary executable.
1082
+
1083
+ #### Inspect
1084
+
1085
+ ```bash
1086
+ forgeloop doctor --json
1087
+ forgeloop index-status --json
1088
+ forgeloop search "pattern" --json
1089
+ ```
1090
+
1091
+ #### Safe recovery
1092
+
1093
+ Retry the search once after confirming that the current package is complete
1094
+ and the managed Repository Index reports `READY`. The client performs at most
1095
+ one bounded transport recovery attempt. If it still fails, preserve the
1096
+ structured error and follow the Repository Index recovery procedure above;
1097
+ transport recovery does not rebuild task state or completion evidence.
1098
+
1099
+ `E_PERSISTENT_TRANSPORT_UNAVAILABLE`,
1100
+ `E_PERSISTENT_TRANSPORT_TIMEOUT`, and
1101
+ `E_PERSISTENT_TRANSPORT_START_FAILED` identify the local transport boundary.
1102
+ The canonical search error, if one is returned after a successful connection,
1103
+ must be diagnosed as a Repository Index or request error instead.
1104
+
1105
+ ### Symptom: Persistent search host ownership is unverified or stale
1106
+
1107
+ #### What it means
1108
+
1109
+ The local state or endpoint does not prove that the live process is the expected
1110
+ ForgeLoop persistent host. Common causes include a host from another package
1111
+ version, a process that exited while state remained, endpoint substitution, or
1112
+ PID reuse. A PID or process name alone is never sufficient ownership evidence.
1113
+
1114
+ #### Inspect
1115
+
1116
+ ```bash
1117
+ forgeloop doctor --json
1118
+ forgeloop search "pattern" --json
1119
+ ```
1120
+
1121
+ The sanitized `persistentTransport` status reports only its public schema,
1122
+ state, running/owned flags, and protocol/package versions. It intentionally
1123
+ does not print endpoint, home, lock, repository, index, binary, or entrypoint
1124
+ paths.
1125
+
1126
+ #### Safe recovery
1127
+
1128
+ Allow the CLI's bounded recovery path to reconcile a stale or incompatible
1129
+ transport state. Explicit shutdown also requires a verified endpoint and nonce.
1130
+ If the status remains `OWNERSHIP_UNVERIFIED`, do not kill a PID or delete a
1131
+ whole `.forgeloop` directory; inspect the structured error and use the normal
1132
+ package/process recovery boundary. The relevant stable codes are
1133
+ `E_PERSISTENT_TRANSPORT_OWNERSHIP_UNVERIFIED`,
1134
+ `E_PERSISTENT_TRANSPORT_HOST_STALE`, and
1135
+ `E_PERSISTENT_TRANSPORT_PROTOCOL_MISMATCH`.
1136
+
1019
1137
  ## Stable Error and Reason Codes
1020
1138
 
1021
1139
  <!-- BEGIN FORGELOOP GENERATED: public-error-codes -->
@@ -1162,6 +1280,16 @@ current-cycle validation.
1162
1280
  | `E_NATIVE_ADAPTER_TARGET_MISSING` | A ForgeLoop protocol validation or lifecycle condition was not satisfied. | Inspect the structured command result, correct the named artifact or prerequisite, then run forgeloop next --json. |
1163
1281
  | `E_NEW_POLICY_VIOLATION` | New executable policy violation detected that is not present in brownfield baseline. | Fix the violation before completing the task. |
1164
1282
  | `E_OBSERVATION_INVALID` | A ForgeLoop protocol validation or lifecycle condition was not satisfied. | Inspect the structured command result, correct the named artifact or prerequisite, then run forgeloop next --json. |
1283
+ | `E_PERSISTENT_TRANSPORT_FRAME_INVALID` | The local persistent-search byte stream did not contain a complete valid JSON frame. | Retry the bounded local request; inspect the host only if malformed frames recur. |
1284
+ | `E_PERSISTENT_TRANSPORT_FRAME_TOO_LARGE` | A persistent-search request or response exceeded its bounded frame limit. | Narrow the search request or inspect the host resource boundary; oversized frames are rejected. |
1285
+ | `E_PERSISTENT_TRANSPORT_HOST_STALE` | Persistent-search state points to a host process that is no longer running. | Retry the CLI search so ForgeLoop can remove only the verified stale host state and restart it. |
1286
+ | `E_PERSISTENT_TRANSPORT_INVALID_REQUEST` | A local persistent-search request is outside the versioned transport contract. | Use the supported ForgeLoop search command or update the compatible client and host together. |
1287
+ | `E_PERSISTENT_TRANSPORT_INVALID_RESPONSE` | The persistent-search host returned a response outside the versioned transport contract. | Retry once through the bounded recovery path; do not consume an unvalidated response. |
1288
+ | `E_PERSISTENT_TRANSPORT_OWNERSHIP_UNVERIFIED` | ForgeLoop could not prove that the process or endpoint belongs to its user-scoped persistent-search host. | Do not terminate the process; inspect the endpoint and retry after resolving the ownership conflict. |
1289
+ | `E_PERSISTENT_TRANSPORT_PROTOCOL_MISMATCH` | The persistent-search client and host do not agree on the supported protocol version. | ForgeLoop may replace only a verified compatible host; otherwise update the installed package and retry. |
1290
+ | `E_PERSISTENT_TRANSPORT_START_FAILED` | The user-scoped persistent-search host could not start or become ready within its bounded startup window. | Inspect the structured diagnostics and retry; direct integration APIs remain available without this optimization. |
1291
+ | `E_PERSISTENT_TRANSPORT_TIMEOUT` | A persistent-search connection, handshake, or request exceeded its bounded timeout. | Retry once through the ownership-checked recovery path and inspect host/index health if it persists. |
1292
+ | `E_PERSISTENT_TRANSPORT_UNAVAILABLE` | The user-scoped persistent-search endpoint was not reachable. | ForgeLoop starts one verified local host and retries once; persistent failure is reported without an rg fallback. |
1165
1293
  | `E_PHASE_CHRONOLOGY_INVALID` | A ForgeLoop protocol validation or lifecycle condition was not satisfied. | Inspect the structured command result, correct the named artifact or prerequisite, then run forgeloop next --json. |
1166
1294
  | `E_PHASE_PREREQUISITE_MISSING` | A ForgeLoop protocol validation or lifecycle condition was not satisfied. | Inspect the structured command result, correct the named artifact or prerequisite, then run forgeloop next --json. |
1167
1295
  | `E_PHASE_TRANSITION_INVALID` | A ForgeLoop protocol validation or lifecycle condition was not satisfied. | Inspect the structured command result, correct the named artifact or prerequisite, then run forgeloop next --json. |
@@ -1208,6 +1336,25 @@ current-cycle validation.
1208
1336
  | `E_RECONCILE_REQUIREMENT_UNKNOWN` | The supplied check id and requirement text do not exactly match a contract verification item of type VERIFICATION. | Supply the exact id and requirement text of an existing contract verification item. |
1209
1337
  | `E_RECONCILE_UNSUPPORTED_DRIFT` | Work-state drift includes kinds other than REPOSITORY_CHANGED (contract or required-artifact drift). | Resolve contract or artifact drift through their dedicated recovery surfaces; reconcile-closure only refreshes repository fingerprint drift. |
1210
1338
  | `E_REPOSITORY_CHANGED` | The repository fingerprint (branch or HEAD) moved after the work-state checkpoint was recorded. | If the task objective is already satisfied in the current repository, run forgeloop reconcile-closure; otherwise resume from a checkpoint that matches the current repository. |
1339
+ | `E_REPOSITORY_INDEX_ENGINE_BINARY_CHECKSUM_MISMATCH` | The extracted or managed tgrep executable bytes do not match the pinned binary SHA-256 digest. | Run forgeloop index-setup or index-rebuild to repair the managed binary; do not run a mismatched executable. |
1340
+ | `E_REPOSITORY_INDEX_ENGINE_CHECKSUM_MISMATCH` | The tgrep archive or executable bytes do not match the pinned manifest SHA-256 digest. | Obtain the exact manifest asset and retry; do not install a checksum mismatch. |
1341
+ | `E_REPOSITORY_INDEX_ENGINE_DOWNLOAD_FAILED` | The pinned tgrep release asset could not be downloaded or read. | Retry with network access or preload the exact manifest archive; never bypass provisioning verification. |
1342
+ | `E_REPOSITORY_INDEX_ENGINE_EXECUTION_FAILED` | A managed tgrep process could not be launched, completed, or stayed within its execution boundary. | Inspect the structured error and index status, then retry with the verified managed engine. |
1343
+ | `E_REPOSITORY_INDEX_ENGINE_EXTRACTION_FAILED` | The tgrep archive is malformed, unsafe, or could not be extracted to a temporary directory. | Use the exact supported archive and retry; unsafe paths and links are rejected. |
1344
+ | `E_REPOSITORY_INDEX_ENGINE_MISSING` | The managed tgrep executable is absent, unreadable, or has not been provisioned. | Run forgeloop index-setup or preload the exact manifest archive with --asset. |
1345
+ | `E_REPOSITORY_INDEX_ENGINE_VERSION_MISMATCH` | The executable reports a version different from the ForgeLoop-pinned tgrep version. | Provision the manifest-pinned tgrep release and retry; do not use an unpinned binary. |
1346
+ | `E_REPOSITORY_INDEX_INDEXING` | The repository index is still being built or reconciled and is not ready for the requested operation. | Wait for index-status to report READY, or use index-rebuild if the operation remains stuck. |
1347
+ | `E_REPOSITORY_INDEX_LOCK_UNSAFE` | A Repository Index operation lock is malformed, conflicting, or cannot be safely acquired. | Wait for a concurrent operation to finish and retry; use status or rebuild for a persistent lock failure. |
1348
+ | `E_REPOSITORY_INDEX_NOT_INITIALIZED` | The repository index has no complete derived index available for the requested operation. | Run forgeloop index-setup or forgeloop index-rebuild for the selected repository. |
1349
+ | `E_REPOSITORY_INDEX_OUTPUT_INVALID` | Native tgrep output did not match the bounded provider-neutral JSON contract. | Treat the result as unusable, inspect the engine, and rebuild or provision the pinned release. |
1350
+ | `E_REPOSITORY_INDEX_OUTPUT_LIMIT` | Managed tgrep output exceeded ForgeLoop's bounded process-output limit. | Narrow the search or resource policy and retry; oversized output is never promoted to a result. |
1351
+ | `E_REPOSITORY_INDEX_PLATFORM_UNSUPPORTED` | The current operating-system and architecture pair has no pinned tgrep release asset. | Use a supported platform or add a separately reviewed manifest asset; do not substitute a PATH executable. |
1352
+ | `E_REPOSITORY_INDEX_REBUILD_FAILED` | The derived repository index could not be rebuilt successfully. | Inspect the structured failure, resource policy, and repository paths, then retry index-rebuild. |
1353
+ | `E_REPOSITORY_INDEX_REQUEST_INVALID` | Repository Index command input is outside the bounded provider-neutral request contract. | Correct the pattern, path filters, context, or lifecycle options and retry the canonical command. |
1354
+ | `E_REPOSITORY_INDEX_SEARCH_FAILED` | The native tgrep search failed with an execution or provider error; no-match is not an error. | Inspect index-status and retry the query or rebuild the derived index; ForgeLoop does not fall back silently. |
1355
+ | `E_REPOSITORY_INDEX_SERVER_START_FAILED` | The ForgeLoop-owned tgrep watcher could not be started or did not become healthy. | Run index-status, inspect the structured reason, and retry index-start or index-rebuild. |
1356
+ | `E_REPOSITORY_INDEX_SERVER_STOP_FAILED` | The ForgeLoop-owned tgrep watcher could not be stopped or its ownership could not be proven. | Use index-status and retry the canonical stop operation; never terminate processes by name or PID alone. |
1357
+ | `E_REPOSITORY_INDEX_SERVER_UNHEALTHY` | Repository Index server, metadata, process identity, or index readiness validation failed. | Run index-status, then use index-rebuild after resolving the reported boundary. |
1211
1358
  | `E_RESPONSIBILITY_FROZEN_INPUT_DRIFT` | A ForgeLoop boundary, artifact, provider, or attestation validation condition was not satisfied. | Inspect the structured command result, correct the named boundary or artifact, then retry the canonical command. |
1212
1359
  | `E_RESPONSIBILITY_INVALID` | A ForgeLoop boundary, artifact, provider, or attestation validation condition was not satisfied. | Inspect the structured command result, correct the named boundary or artifact, then retry the canonical command. |
1213
1360
  | `E_RESPONSIBILITY_REQUIRED_CHECK_MISSING` | A ForgeLoop boundary, artifact, provider, or attestation validation condition was not satisfied. | Inspect the structured command result, correct the named boundary or artifact, then retry the canonical command. |
@@ -67,6 +67,36 @@ may be absent only when the canonical protocol marks it not applicable;
67
67
  presentation depth cannot change evidence, verification truth, authority,
68
68
  provenance, safety-floor, or validator-backed completion requirements.
69
69
 
70
+ ### Repository Search boundary
71
+
72
+ The Integration API exposes direct, transport-neutral repository operations:
73
+
74
+ ```js
75
+ import {
76
+ repositorySearch,
77
+ repositoryIndexStatus,
78
+ } from "@cassiomc1/forgeloop/integration";
79
+
80
+ const result = await repositorySearch({
81
+ projectPath: ".",
82
+ pattern: "needle",
83
+ fixedStrings: true,
84
+ });
85
+
86
+ const status = await repositoryIndexStatus({ projectPath: "." });
87
+ ```
88
+
89
+ These operations call the canonical Repository Search service and require the
90
+ managed Repository Index for supported Git repositories. The shell-facing CLI
91
+ `search` command may use the user-scoped persistent local IPC host described in
92
+ [`PERSISTENT_SEARCH_TRANSPORT.md`](./PERSISTENT_SEARCH_TRANSPORT.md), but that
93
+ host is only a CLI startup/reuse optimization. Integration API and MCP callers
94
+ do not use it, and no transport carries lifecycle, claims, receipts, evidence,
95
+ or completion authority.
96
+
97
+ See [`REPOSITORY_INDEX.md`](./REPOSITORY_INDEX.md) for the normalized search
98
+ contract, tgrep management, status resource, and maintenance commands.
99
+
70
100
  ### Usage and efficiency boundary
71
101
 
72
102
  Create a context with an optional trusted provider:
@@ -87,3 +87,13 @@ These explanations are part of the JSON sources and generated explorers;
87
87
  matching textual fallbacks live in the README, revision-provider guide, and
88
88
  attestation guide. Re-render before renewing a visual review, then inspect
89
89
  the new output at desktop and narrow widths before binding the review hashes.
90
+
91
+ The Repository Index and Persistent Search Transport are runtime discovery
92
+ boundaries rather than lifecycle transitions. Their current CLI/API/MCP
93
+ separation and local IPC sequence are documented in
94
+ [`REPOSITORY_INDEX.md`](../REPOSITORY_INDEX.md) and
95
+ [`PERSISTENT_SEARCH_TRANSPORT.md`](../PERSISTENT_SEARCH_TRANSPORT.md). They are
96
+ intentionally not added as a fourth Archify workflow merely to increase the
97
+ diagram count; the README hero provides the conceptual architecture overview,
98
+ while these three governed visuals remain focused on lifecycle, verification,
99
+ and attestation.
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@cassiomc1/forgeloop",
3
- "version": "1.10.2",
3
+ "version": "1.11.0",
4
4
  "description": "Portable, verifiable engineering protocol for AI coding environments and developer workflows",
5
5
  "repository": {
6
6
  "type": "git",
@@ -24,6 +24,10 @@
24
24
  "schemas",
25
25
  "benchmarks/execution-profiles/*.json",
26
26
  "benchmarks/execution-profiles/README.md",
27
+ "benchmarks/repository-index/queries.json",
28
+ "benchmarks/repository-index/README.md",
29
+ "benchmarks/repository-index/run-hot-path.mjs",
30
+ "benchmarks/repository-index/run-persistent-transport.mjs",
27
31
  ".forgeloop/forgeloop.gitignore",
28
32
  "AGENTS.md",
29
33
  "CLAUDE.md",
@@ -72,6 +76,8 @@
72
76
  "docs/PLATFORM_ADAPTERS.md",
73
77
  "docs/AGENT_PROTOCOL_SUMMARY.md",
74
78
  "docs/EXECUTION_PROFILE_BENCHMARKS.md",
79
+ "docs/REPOSITORY_INDEX.md",
80
+ "docs/PERSISTENT_SEARCH_TRANSPORT.md",
75
81
  "docs/KNOWLEDGE_SOURCES.md",
76
82
  "scripts/generate-shell-completions.mjs",
77
83
  "scripts/generate-agent-protocol-summary.mjs",
@@ -80,6 +86,8 @@
80
86
  "scripts/validate-execution-profile-benchmarks.mjs",
81
87
  "scripts/report-execution-profile-outliers.mjs",
82
88
  "scripts/report-tail-interpretation.mjs",
89
+ "scripts/verify-tgrep-manifest.mjs",
90
+ "scripts/update-tgrep-manifest.mjs",
83
91
  "scripts/check-efficiency-regression.mjs",
84
92
  "scripts/lib/execution-profile-benchmark-io.mjs",
85
93
  "scripts/benchmark-cli-startup.mjs",
@@ -131,7 +139,8 @@
131
139
  "benchmark:profiles:tail-analysis": "node scripts/report-tail-interpretation.mjs",
132
140
  "benchmark:profiles:regression": "node scripts/check-efficiency-regression.mjs",
133
141
  "transactions:compact": "node scripts/compact-transactions.mjs",
134
- "complexity:check": "node scripts/check-complexity.mjs"
142
+ "complexity:check": "node scripts/check-complexity.mjs",
143
+ "repository-index:manifest": "node scripts/verify-tgrep-manifest.mjs"
135
144
  },
136
145
  "devDependencies": {
137
146
  "c8": "^12.0.0",
@@ -0,0 +1,86 @@
1
+ import { execFile as nodeExecFile } from "node:child_process";
2
+ import { mkdtemp, readFile, readdir, rm, stat, writeFile } from "node:fs/promises";
3
+ import os from "node:os";
4
+ import path from "node:path";
5
+ import { fileURLToPath } from "node:url";
6
+
7
+ import { validateTgrepManifest } from "../src/repository-index/manifest.js";
8
+ import { sha256File, validateArchiveEntry } from "../src/repository-index/binary-manager.js";
9
+
10
+ const packageRoot = path.resolve(path.dirname(fileURLToPath(import.meta.url)), "..");
11
+ const manifestPath = path.join(packageRoot, "src", "repository-index", "tgrep-manifest.json");
12
+ const args = process.argv.slice(2);
13
+ const assetDirIndex = args.indexOf("--asset-dir");
14
+ const assetDir = assetDirIndex >= 0 ? args[assetDirIndex + 1] : null;
15
+ const shouldWrite = args.includes("--write");
16
+
17
+ if (!assetDir || !path.isAbsolute(assetDir) || !shouldWrite) {
18
+ console.error("Usage: node scripts/update-tgrep-manifest.mjs --asset-dir /absolute/release-assets --write");
19
+ process.exit(1);
20
+ }
21
+
22
+ const manifest = JSON.parse(await readFile(manifestPath, "utf8"));
23
+ validateTgrepManifest(manifest);
24
+
25
+ function execFile(file, args, options = {}) {
26
+ return new Promise((resolve, reject) => {
27
+ nodeExecFile(file, args, { shell: false, ...options }, (error, stdout, stderr) => {
28
+ if (error) {
29
+ error.stdout = stdout;
30
+ error.stderr = stderr;
31
+ reject(error);
32
+ return;
33
+ }
34
+ resolve({ stdout, stderr });
35
+ });
36
+ });
37
+ }
38
+
39
+ async function listArchiveEntries(archivePath, archive) {
40
+ const command = archive === "zip" && process.platform !== "win32" ? "unzip" : "tar";
41
+ const args = command === "unzip" ? ["-Z1", archivePath] : ["-tf", archivePath];
42
+ const result = await execFile(command, args, { maxBuffer: 2 * 1024 * 1024 });
43
+ return result.stdout.split(/\r?\n/).filter(Boolean).map(validateArchiveEntry);
44
+ }
45
+
46
+ async function extractBinary(assetPath, asset, destination) {
47
+ const entries = await listArchiveEntries(assetPath, asset.archive);
48
+ if (!entries.some((entry) => path.posix.basename(entry) === asset.binaryName)) {
49
+ throw new Error(`archive does not contain ${asset.binaryName}`);
50
+ }
51
+ const command = asset.archive === "zip" && process.platform !== "win32" ? "unzip" : "tar";
52
+ const args = command === "unzip"
53
+ ? ["-q", assetPath, "-d", destination]
54
+ : ["-xf", assetPath, "-C", destination];
55
+ await execFile(command, args, { maxBuffer: 2 * 1024 * 1024 });
56
+
57
+ const found = [];
58
+ async function visit(directory) {
59
+ for (const entry of await readdir(directory, { withFileTypes: true })) {
60
+ const entryPath = path.join(directory, entry.name);
61
+ if (entry.isSymbolicLink()) throw new Error(`archive extracted a symbolic link: ${entry.name}`);
62
+ if (entry.isDirectory()) await visit(entryPath);
63
+ else if (entry.isFile() && entry.name === asset.binaryName) found.push(entryPath);
64
+ }
65
+ }
66
+ await visit(destination);
67
+ if (found.length !== 1) throw new Error(`expected exactly one ${asset.binaryName}, found ${found.length}`);
68
+ return found[0];
69
+ }
70
+
71
+ for (const asset of Object.values(manifest.assets)) {
72
+ const assetPath = path.join(assetDir, asset.assetName);
73
+ const info = await stat(assetPath);
74
+ if (!info.isFile()) throw new Error(`asset is not a regular file: ${assetPath}`);
75
+ asset.sha256 = await sha256File(assetPath);
76
+ const extractionDirectory = await mkdtemp(path.join(os.tmpdir(), "forgeloop-tgrep-manifest-"));
77
+ try {
78
+ const extractedBinary = await extractBinary(assetPath, asset, extractionDirectory);
79
+ asset.binarySha256 = await sha256File(extractedBinary);
80
+ } finally {
81
+ await rm(extractionDirectory, { recursive: true, force: true });
82
+ }
83
+ }
84
+ validateTgrepManifest(manifest);
85
+ await writeFile(manifestPath, `${JSON.stringify(manifest, null, 2)}\n`);
86
+ console.log(`updated ${manifestPath}`);
@@ -0,0 +1,17 @@
1
+ import { readFile } from "node:fs/promises";
2
+ import path from "node:path";
3
+ import { fileURLToPath } from "node:url";
4
+
5
+ import { validateTgrepManifest } from "../src/repository-index/manifest.js";
6
+
7
+ const packageRoot = path.resolve(path.dirname(fileURLToPath(import.meta.url)), "..");
8
+ const manifestPath = path.join(packageRoot, "src", "repository-index", "tgrep-manifest.json");
9
+
10
+ try {
11
+ const value = JSON.parse(await readFile(manifestPath, "utf8"));
12
+ validateTgrepManifest(value);
13
+ console.log(`valid tgrep manifest: ${value.engine} ${value.version} (${Object.keys(value.assets).length} assets)`);
14
+ } catch (error) {
15
+ console.error(`invalid tgrep manifest: ${error.message}`);
16
+ process.exitCode = 1;
17
+ }
package/src/cli.js CHANGED
@@ -85,6 +85,11 @@ import { formatAttestationCreateResult } from "./commands/attestation-create.js"
85
85
  import { formatAttestationVerifyResult } from "./commands/attestation-verify.js";
86
86
  import { formatAttestationStatusResult } from "./commands/attestation-status.js";
87
87
  import { formatAttestationVerifyRangeResult } from "./commands/attestation-verify-range.js";
88
+ import {
89
+ formatRepositoryIndexResult,
90
+ formatRepositoryIndexStatus,
91
+ formatSearchResult,
92
+ } from "./commands/repository-index.js";
88
93
  import { defaultCommandInputValues, validateForgeLoopCommandInput } from "./core/command-input.js";
89
94
  import { COMMAND_EXECUTORS } from "./core/command-executors.js";
90
95
  import { resolveTarget } from "./core/filesystem.js";
@@ -368,6 +373,36 @@ export const COMMAND_HANDLERS = Object.freeze({
368
373
  }
369
374
  return result.ok ? 0 : 1;
370
375
  },
376
+ "index-setup": async ({ target, packageRoot, options }) => {
377
+ const { result } = await COMMAND_EXECUTORS["index-setup"]({ target, packageRoot, options });
378
+ renderJsonOr(options, result, formatRepositoryIndexResult);
379
+ return 0;
380
+ },
381
+ "index-start": async ({ target, packageRoot, options }) => {
382
+ const { result } = await COMMAND_EXECUTORS["index-start"]({ target, packageRoot, options });
383
+ renderJsonOr(options, result, formatRepositoryIndexResult);
384
+ return 0;
385
+ },
386
+ "index-stop": async ({ target, packageRoot, options }) => {
387
+ const { result } = await COMMAND_EXECUTORS["index-stop"]({ target, packageRoot, options });
388
+ renderJsonOr(options, result, formatRepositoryIndexResult);
389
+ return 0;
390
+ },
391
+ "index-status": async ({ target, packageRoot, options }) => {
392
+ const { result } = await COMMAND_EXECUTORS["index-status"]({ target, packageRoot, options });
393
+ renderJsonOr(options, result, formatRepositoryIndexStatus);
394
+ return 0;
395
+ },
396
+ "index-rebuild": async ({ target, packageRoot, options }) => {
397
+ const { result } = await COMMAND_EXECUTORS["index-rebuild"]({ target, packageRoot, options });
398
+ renderJsonOr(options, result, formatRepositoryIndexResult);
399
+ return 0;
400
+ },
401
+ search: async ({ target, packageRoot, options }) => {
402
+ const { result, exitCode } = await COMMAND_EXECUTORS.search({ target, packageRoot, options, transport: "cli" });
403
+ renderJsonOr(options, result, formatSearchResult);
404
+ return exitCode;
405
+ },
371
406
  route: async ({ target, packageRoot, options }) => {
372
407
  const { result } = await COMMAND_EXECUTORS.route({ target, packageRoot, options });
373
408
  renderJsonOr(options, result, formatRouteResult);
@@ -5,6 +5,10 @@ import { createEvidence } from "../core/evidence.js";
5
5
  import { LAYOUT_VERSION } from "../core/target-layout.js";
6
6
  import { inspectNativeAdapter, validateNativeAdapterTargets } from "../core/native-adapters.js";
7
7
  import { findIncompleteTransactions, recoverIncompleteTransactions } from "../core/transaction.js";
8
+ import { isRepositoryCandidate } from "../repository-index/lifecycle.js";
9
+ import { getRepositoryIndexStatus, sanitizeRepositoryIndexStatus } from "../repository-index/status.js";
10
+ import { searchRepository } from "../repository-index/search.js";
11
+ import { getPersistentTransportStatus } from "../persistent-transport/client.js";
8
12
 
9
13
  function finding(code, severity, relativePath, message, remediation = null, evidence = null) {
10
14
  const evidenceRecord = evidence && typeof evidence === "object"
@@ -114,7 +118,69 @@ async function adoptAdapters({ target, manifest, adoptPaths, findings }) {
114
118
  return nextManifest;
115
119
  }
116
120
 
117
- export async function runDoctor({ target, packageRoot, adoptPaths = [], strict = false, fix = false }) {
121
+ async function inspectRepositoryIndexForDoctor({ target, packageRoot, repositoryIndex, repositoryIndexOptions, findings }) {
122
+ const persistentTransport = await getPersistentTransportStatus({ homeDirectory: repositoryIndexOptions?.homeDirectory });
123
+ if (!repositoryIndex) return { status: "DISABLED", required: false, persistentTransport };
124
+ if (!(await isRepositoryCandidate(target))) {
125
+ return { status: "DEFERRED", required: true, reason: "target is not a Git repository", persistentTransport };
126
+ }
127
+
128
+ const status = await getRepositoryIndexStatus(target, { packageRoot, ...repositoryIndexOptions, includeLocalPaths: true });
129
+ const publicStatus = sanitizeRepositoryIndexStatus(status);
130
+ const result = {
131
+ status: publicStatus.health,
132
+ required: true,
133
+ engine: publicStatus.engine,
134
+ engineVersion: publicStatus.engineVersion,
135
+ indexPath: ".forgeloop/repository-index/tgrep",
136
+ server: publicStatus.server,
137
+ diagnostics: publicStatus.diagnostics,
138
+ persistentTransport,
139
+ };
140
+ if (status.health === "READY") {
141
+ try {
142
+ const smoke = await searchRepository(target, {
143
+ ...repositoryIndexOptions,
144
+ packageRoot,
145
+ pattern: "__FORGELOOP_DOCTOR_INDEX_SMOKE__",
146
+ fixedStrings: true,
147
+ maxCount: 1,
148
+ });
149
+ result.smokeSearch = {
150
+ status: "OK",
151
+ matchCount: smoke.matches.length,
152
+ nativeExitCode: smoke.metrics.exitCode,
153
+ };
154
+ } catch (error) {
155
+ result.smokeSearch = {
156
+ status: "ERROR",
157
+ code: error.code ?? "E_REPOSITORY_INDEX_SEARCH_FAILED",
158
+ message: error.message,
159
+ };
160
+ findings.push(finding(
161
+ error.code ?? "E_REPOSITORY_INDEX_SEARCH_FAILED",
162
+ "error",
163
+ ".forgeloop/repository-index",
164
+ `Repository Index smoke search failed: ${error.message}`,
165
+ "Run forgeloop index-status --json, then index-rebuild if the managed search service is unhealthy.",
166
+ error.message,
167
+ ));
168
+ }
169
+ }
170
+ if (status.health !== "READY") {
171
+ findings.push(finding(
172
+ publicStatus.diagnostics?.[0]?.code ?? "E_REPOSITORY_INDEX_SERVER_UNHEALTHY",
173
+ "error",
174
+ ".forgeloop/repository-index",
175
+ `Repository Index is ${status.health}; ${publicStatus.diagnostics?.[0]?.message ?? "native index health is not ready"}`,
176
+ "Run forgeloop index-setup or forgeloop index-rebuild with an approved managed tgrep asset.",
177
+ publicStatus,
178
+ ));
179
+ }
180
+ return result;
181
+ }
182
+
183
+ export async function runDoctor({ target, packageRoot, adoptPaths = [], strict = false, fix = false, repositoryIndex, repositoryIndexOptions }) {
118
184
  const findings = [];
119
185
  let incompleteTransactions = await findIncompleteTransactions(target);
120
186
  if (fix && incompleteTransactions.some((transaction) => transaction.status === "COMMITTING")) {
@@ -318,11 +384,20 @@ export async function runDoctor({ target, packageRoot, adoptPaths = [], strict =
318
384
  await writeManifest(target, manifest);
319
385
  }
320
386
 
387
+ const repositoryIndexResult = await inspectRepositoryIndexForDoctor({
388
+ target,
389
+ packageRoot,
390
+ repositoryIndex,
391
+ repositoryIndexOptions,
392
+ findings,
393
+ });
394
+
321
395
  const ok = findings.every((item) => item.severity !== "error")
322
396
  && (!strict || findings.every((item) => item.severity !== "warning"));
323
397
  return {
324
398
  ok,
325
399
  findings,
400
+ repositoryIndex: repositoryIndexResult,
326
401
  evidence: [createEvidence({
327
402
  kind: "OBSERVED",
328
403
  source: "ForgeLoop doctor",
@@ -0,0 +1 @@
1
+ export { runRepositoryIndexRebuild, formatRepositoryIndexResult } from "./repository-index.js";
@@ -0,0 +1 @@
1
+ export { runRepositoryIndexSetup, formatRepositoryIndexResult } from "./repository-index.js";
@@ -0,0 +1 @@
1
+ export { runRepositoryIndexStart, formatRepositoryIndexResult } from "./repository-index.js";
@@ -0,0 +1 @@
1
+ export { runRepositoryIndexStatus, formatRepositoryIndexStatus } from "./repository-index.js";
@@ -0,0 +1 @@
1
+ export { runRepositoryIndexStop, formatRepositoryIndexResult } from "./repository-index.js";
@@ -9,6 +9,8 @@ import { readTemplateEntries } from "../core/templates.js";
9
9
  import { PROJECT_ARTIFACT_PATHS } from "../core/task-paths.js";
10
10
  import { isKitPath } from "../core/target-layout.js";
11
11
  import { E_INIT_KIT_CONFLICT, E_POLICY_INITIALIZATION_FAILED } from "../core/error-codes.js";
12
+ import { isRepositoryCandidate } from "../repository-index/lifecycle.js";
13
+ import { setupRepositoryIndex } from "../repository-index/server.js";
12
14
 
13
15
  // Compatibility re-exports: the canonical semantic definitions live in
14
16
  // src/core/error-codes.js; this keeps existing import paths working while
@@ -107,6 +109,31 @@ function kitConflictError(relativePath) {
107
109
  return error;
108
110
  }
109
111
 
112
+ async function prepareRepositoryIndexForInit({ target, dryRun, packageRoot, repositoryIndex, repositoryIndexOptions, actions }) {
113
+ if (!repositoryIndex) return { status: "DISABLED", required: false };
114
+ if (!(await isRepositoryCandidate(target))) {
115
+ return { status: "DEFERRED", required: true, reason: "target is not a Git repository" };
116
+ }
117
+ if (dryRun) {
118
+ return {
119
+ status: "WOULD_SETUP",
120
+ required: true,
121
+ reason: "dry-run does not provision or execute the native index engine",
122
+ };
123
+ }
124
+
125
+ const setup = await setupRepositoryIndex(target, { ...repositoryIndexOptions, packageRoot });
126
+ actions.push({ action: "repository-index-ready", path: ".forgeloop/repository-index", reason: "managed tgrep index and watcher are ready" });
127
+ return {
128
+ status: "READY",
129
+ required: true,
130
+ health: setup.status?.health ?? "READY",
131
+ engine: setup.status?.engine ?? "tgrep",
132
+ engineVersion: setup.status?.engineVersion ?? null,
133
+ server: setup.status?.server ?? null,
134
+ };
135
+ }
136
+
110
137
  /**
111
138
  * Initializes a target project with the ForgeLoop kit and executable-policy
112
139
  * bootstrap, committing manifest authority LAST.
@@ -152,6 +179,8 @@ export async function runInit({
152
179
  packageRoot,
153
180
  packageVersion,
154
181
  hooks = {},
182
+ repositoryIndex,
183
+ repositoryIndexOptions,
155
184
  } = {}) {
156
185
  const entries = await readTemplateEntries(packageRoot);
157
186
  const existingManifest = await readManifest(target);
@@ -332,6 +361,15 @@ export async function runInit({
332
361
  }
333
362
  }
334
363
 
364
+ const repositoryIndexResult = await prepareRepositoryIndexForInit({
365
+ target,
366
+ dryRun,
367
+ packageRoot,
368
+ repositoryIndex,
369
+ repositoryIndexOptions,
370
+ actions,
371
+ });
372
+
335
373
  // 10. Commit manifest authority LAST: successful initialization is only
336
374
  // real once the manifest exists.
337
375
  if (hooks.beforeManifestWrite) {
@@ -343,5 +381,5 @@ export async function runInit({
343
381
  }
344
382
  await writeManifest(target, manifest, { dryRun });
345
383
 
346
- return { actions, manifest };
384
+ return { actions, manifest, repositoryIndex: repositoryIndexResult };
347
385
  }