@haystackeditor/cli 0.25.0 → 0.26.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (113) hide show
  1. package/README.md +39 -455
  2. package/dist/capture/adapters/django.js +7 -0
  3. package/dist/capture/adapters/index.js +10 -0
  4. package/dist/capture/adapters/next-build.js +113 -0
  5. package/dist/capture/adapters/next.js +272 -105
  6. package/dist/capture/adapters/nuxt.js +7 -0
  7. package/dist/capture/adapters/rails.js +7 -0
  8. package/dist/capture/adapters/react-router.js +7 -0
  9. package/dist/capture/adapters/sveltekit.js +7 -0
  10. package/dist/capture/adapters/vite.js +14 -0
  11. package/dist/capture/app-config.js +27 -2
  12. package/dist/capture/csp.js +7 -6
  13. package/dist/capture/url-rewrites.js +232 -0
  14. package/dist/capture-step.js +3 -0
  15. package/dist/commands/capture-brief.js +41 -32
  16. package/dist/commands/capture-manifest.js +25 -17
  17. package/dist/commands/feedback.js +66 -0
  18. package/dist/commands/init-capture.js +44 -27
  19. package/dist/commands/init-telemetry.js +77 -14
  20. package/dist/commands/init.js +19 -4
  21. package/dist/commands/lockfile-pin.js +307 -0
  22. package/dist/commands/verify-onboarding.js +16 -0
  23. package/dist/commands/verify.js +70 -15
  24. package/dist/index.js +29 -923
  25. package/dist/schema.js +4 -10
  26. package/dist/utils/haystack-api.js +0 -36
  27. package/package.json +1 -5
  28. package/schemas/feedback.v1.json +13 -0
  29. package/schemas/init.v1.json +1 -1
  30. package/schemas/pre-verify.v2.json +240 -0
  31. package/schemas/verify-raw.v1.json +1132 -0
  32. package/schemas/verify.v2.json +655 -0
  33. package/dist/assets/hooks/agent-context/detect.ts +0 -316
  34. package/dist/assets/hooks/agent-context/format.ts +0 -100
  35. package/dist/assets/hooks/agent-context/index.ts +0 -41
  36. package/dist/assets/hooks/agent-context/parsers/claude.ts +0 -262
  37. package/dist/assets/hooks/agent-context/parsers/codex.ts +0 -416
  38. package/dist/assets/hooks/agent-context/parsers/gemini.ts +0 -155
  39. package/dist/assets/hooks/agent-context/parsers/opencode.ts +0 -174
  40. package/dist/assets/hooks/agent-context/tsconfig.json +0 -14
  41. package/dist/assets/hooks/agent-context/types.ts +0 -58
  42. package/dist/assets/hooks/llm-rules-template.md +0 -59
  43. package/dist/assets/hooks/package-lock.json +0 -598
  44. package/dist/assets/hooks/package.json +0 -12
  45. package/dist/assets/hooks/scripts/commit-msg.sh +0 -5
  46. package/dist/assets/hooks/scripts/post-commit.sh +0 -5
  47. package/dist/assets/hooks/scripts/pre-commit.sh +0 -175
  48. package/dist/assets/hooks/scripts/pre-push.sh +0 -25
  49. package/dist/assets/hooks/scripts/prepare-commit-msg.sh +0 -5
  50. package/dist/assets/hooks/truncation-checker/ast-analyzer.ts +0 -528
  51. package/dist/assets/hooks/truncation-checker/index.ts +0 -595
  52. package/dist/assets/hooks/truncation-checker/tsconfig.json +0 -13
  53. package/dist/assets/skills/map-cloud-verifier-universe/SKILL.md +0 -2051
  54. package/dist/assets/skills/map-cloud-verifier-universe/agents/openai.yaml +0 -4
  55. package/dist/assets/skills/map-cloud-verifier-universe/references/output-contract.md +0 -3411
  56. package/dist/assets/skills/map-your-system.md +0 -143
  57. package/dist/assets/skills/submit.md +0 -200
  58. package/dist/commands/ask.js +0 -20
  59. package/dist/commands/cloud-verifier-behaviors.js +0 -218
  60. package/dist/commands/cloud-verifier-data-store-census.js +0 -539
  61. package/dist/commands/cloud-verifier-data-store-drift.js +0 -158
  62. package/dist/commands/cloud-verifier-identity-census.js +0 -4060
  63. package/dist/commands/cloud-verifier-materialization.js +0 -704
  64. package/dist/commands/cloud-verifier-pascal-selector-census.js +0 -1382
  65. package/dist/commands/cloud-verifier-python-manifest-selector-census.js +0 -2015
  66. package/dist/commands/cloud-verifier-specialized-operational-census.js +0 -11432
  67. package/dist/commands/cloud-verifier-universe.js +0 -10178
  68. package/dist/commands/config.js +0 -549
  69. package/dist/commands/design-verify.js +0 -311
  70. package/dist/commands/dismiss.js +0 -159
  71. package/dist/commands/hooks.js +0 -226
  72. package/dist/commands/inbox.js +0 -137
  73. package/dist/commands/mcp.js +0 -201
  74. package/dist/commands/policy.js +0 -371
  75. package/dist/commands/pr-status.js +0 -207
  76. package/dist/commands/pr.js +0 -105
  77. package/dist/commands/prepare-universe-review.js +0 -1092
  78. package/dist/commands/production-source-deny-policy.js +0 -100
  79. package/dist/commands/request-review.js +0 -74
  80. package/dist/commands/review.js +0 -191
  81. package/dist/commands/rules.js +0 -98
  82. package/dist/commands/scaffold-provisional-universe.js +0 -806
  83. package/dist/commands/setup.js +0 -1170
  84. package/dist/commands/skills.js +0 -447
  85. package/dist/commands/submit.js +0 -745
  86. package/dist/commands/system-map.js +0 -228
  87. package/dist/commands/triage.js +0 -598
  88. package/dist/commands/webhooks.js +0 -241
  89. package/dist/states.js +0 -46
  90. package/dist/tools/detect.js +0 -832
  91. package/dist/triage/astra.js +0 -202
  92. package/dist/triage/prompts.js +0 -188
  93. package/dist/triage/runner.js +0 -200
  94. package/dist/triage/types.js +0 -7
  95. package/dist/utils/action-output.js +0 -26
  96. package/dist/utils/analysis-api.js +0 -416
  97. package/dist/utils/design-verifier-api.js +0 -294
  98. package/dist/utils/design-verifier-history.js +0 -79
  99. package/dist/utils/design-verifier-result.js +0 -424
  100. package/dist/utils/github-api.js +0 -324
  101. package/dist/utils/pending-state.js +0 -86
  102. package/dist/utils/pr-ref.js +0 -56
  103. package/dist/utils/prompter.js +0 -328
  104. package/schemas/action.v1.json +0 -22
  105. package/schemas/ask.v1.json +0 -40
  106. package/schemas/inbox.v1.json +0 -27
  107. package/schemas/pr-status.v1.json +0 -61
  108. package/schemas/pr.v1.json +0 -97
  109. package/schemas/pr.v3.json +0 -45
  110. package/schemas/setup.v1.json +0 -75
  111. package/schemas/submit.v1.json +0 -90
  112. package/schemas/triage.v1.json +0 -103
  113. package/schemas/triage.v2.json +0 -64
@@ -1,539 +0,0 @@
1
- /**
2
- * Data-store census.
3
- *
4
- * The one job the Cloud Verifier needs from a repository map: what data stores
5
- * does this application use, so we can clone each one into a replica. Not
6
- * processes, not API calls, not code structure — the databases, caches, object
7
- * stores, search indexes, and queues that hold state a test must run against.
8
- *
9
- * Four source signals, strongest first:
10
- * 1. a service image in a compose/k8s manifest (`image: postgres:16`) — a
11
- * store instance the app is wired to run beside, the surest signal;
12
- * 2. a connection URL scheme or conventional env var (`postgres://`,
13
- * `REDIS_URL`) — an instance named in config;
14
- * 3. a config declaration that names a store without an image, library, or URL
15
- * (Django `ENGINE`, PHP PDO, Laravel `DB_CONNECTION`, Rails `adapter:`);
16
- * 4. a client library in a dependency manifest (`pg`, `redis`, `mongoose`) —
17
- * the app talks to that engine.
18
- *
19
- * Signals are folded per engine into one data store, with the evidence, a
20
- * confidence, and how to reproduce it. Everything here is a general convention,
21
- * never a repository-specific constant.
22
- */
23
- // How a replica of this category is made. Durable stores of record are copied
24
- // from a snapshot; caches and search indexes are usually derived and can be
25
- // rebuilt; queues are drained to empty.
26
- const CLONE_STRATEGY_BY_CATEGORY = {
27
- 'relational-database': 'snapshot-restore',
28
- 'document-database': 'snapshot-restore',
29
- 'key-value-store': 'snapshot-restore',
30
- 'wide-column-database': 'snapshot-restore',
31
- 'columnar-database': 'snapshot-restore',
32
- 'graph-database': 'snapshot-restore',
33
- 'vector-database': 'snapshot-restore',
34
- 'object-storage': 'copy-bucket-snapshot',
35
- 'search-index': 'rebuild-from-source',
36
- 'message-queue': 'reset-empty',
37
- 'time-series-database': 'snapshot-restore',
38
- 'embedded-database': 'copy-file',
39
- };
40
- // Client library (as written in a dependency manifest) -> engine. Covers the
41
- // common stores across Node, Python, Go, Ruby, Java, PHP. Keys are matched
42
- // case-insensitively against exact dependency names.
43
- // Null-prototype: keys are matched against arbitrary dependency tokens from a
44
- // manifest, so a token like `constructor`/`toString`/`hasOwnProperty` must NOT
45
- // resolve to an inherited Object.prototype member (that returned a truthy
46
- // non-profile and created a store with an undefined engine — crashed on a repo
47
- // whose manifest contained such a token).
48
- const LIBRARY_ENGINES = Object.create(null);
49
- function registerLibraries(profile, names) {
50
- for (const name of names)
51
- LIBRARY_ENGINES[name.toLowerCase()] = profile;
52
- }
53
- registerLibraries({ engine: 'postgres', category: 'relational-database' }, [
54
- 'pg', 'pg-promise', 'postgres', 'node-postgres', 'psycopg2', 'psycopg2-binary',
55
- 'psycopg', 'asyncpg', 'lib/pq', 'github.com/lib/pq', 'github.com/jackc/pgx',
56
- 'github.com/jackc/pgx/v5', 'pg gem', 'postgresql', 'org.postgresql:postgresql',
57
- ]);
58
- registerLibraries({ engine: 'mysql', category: 'relational-database' }, [
59
- 'mysql', 'mysql2', 'mariadb', 'pymysql', 'mysqlclient', 'aiomysql',
60
- 'github.com/go-sql-driver/mysql', 'mysql-connector-python', 'mysql:mysql-connector-java',
61
- ]);
62
- registerLibraries({ engine: 'sqlserver', category: 'relational-database' }, [
63
- 'mssql', 'tedious', 'pyodbc', 'pymssql', 'github.com/denisenkom/go-mssqldb',
64
- ]);
65
- registerLibraries({ engine: 'oracle', category: 'relational-database' }, [
66
- 'oracledb', 'cx_oracle', 'cx-oracle',
67
- ]);
68
- registerLibraries({ engine: 'mongodb', category: 'document-database' }, [
69
- 'mongodb', 'mongoose', 'pymongo', 'motor', 'mongoengine',
70
- 'go.mongodb.org/mongo-driver', 'mongo', 'mongoid',
71
- ]);
72
- registerLibraries({ engine: 'couchdb', category: 'document-database' }, [
73
- 'nano', 'couchdb', 'pycouchdb', 'cloudant',
74
- ]);
75
- registerLibraries({ engine: 'redis', category: 'key-value-store' }, [
76
- 'redis', 'ioredis', 'redis-py', 'connect-redis', 'go-redis', 'github.com/redis/go-redis',
77
- 'github.com/go-redis/redis', 'hiredis', 'aioredis', 'redis-rb', 'predis/predis', 'jedis',
78
- ]);
79
- registerLibraries({ engine: 'memcached', category: 'key-value-store' }, [
80
- 'memcached', 'memjs', 'pymemcache', 'python-memcached', 'dalli',
81
- ]);
82
- registerLibraries({ engine: 'dynamodb', category: 'key-value-store' }, [
83
- '@aws-sdk/client-dynamodb', 'dynamoose',
84
- ]);
85
- registerLibraries({ engine: 'cassandra', category: 'wide-column-database' }, [
86
- 'cassandra-driver', 'cassandra', 'scylla-driver', 'datastax',
87
- ]);
88
- registerLibraries({ engine: 'clickhouse', category: 'columnar-database' }, [
89
- '@clickhouse/client', '@clickhouse/client-web', 'clickhouse-driver',
90
- 'clickhouse-connect', 'clickhouse', 'ecto_ch', 'clickhouse_ecto',
91
- 'github.com/clickhouse/clickhouse-go', 'clickhouse-sqlalchemy',
92
- ]);
93
- registerLibraries({ engine: 'cockroachdb', category: 'relational-database' }, [
94
- 'cockroachdb', 'sqlalchemy-cockroachdb',
95
- ]);
96
- registerLibraries({ engine: 'neo4j', category: 'graph-database' }, [
97
- 'neo4j-driver', 'neo4j', 'py2neo', 'neo4j-python-driver',
98
- ]);
99
- registerLibraries({ engine: 'weaviate', category: 'vector-database' }, [
100
- 'weaviate-client', 'weaviate-ts-client', '@weaviate/client', 'weaviate',
101
- ]);
102
- registerLibraries({ engine: 'qdrant', category: 'vector-database' }, [
103
- 'qdrant-client', '@qdrant/js-client-rest', 'qdrant',
104
- ]);
105
- registerLibraries({ engine: 'milvus', category: 'vector-database' }, [
106
- 'pymilvus', '@zilliz/milvus2-sdk-node', 'milvus',
107
- ]);
108
- registerLibraries({ engine: 'pinecone', category: 'vector-database' }, [
109
- 'pinecone-client', '@pinecone-database/pinecone', 'pinecone',
110
- ]);
111
- registerLibraries({ engine: 'chroma', category: 'vector-database' }, [
112
- 'chromadb', 'chromadb-client',
113
- ]);
114
- // Object-storage clients must be the specific S3 client, not a monolithic
115
- // cloud SDK: bare `aws-sdk`/`boto3`/`aws-sdk-go` are used for SES, SNS, SQS,
116
- // and dozens of non-storage services, so mapping them to S3 fabricates stores
117
- // (measured on medusa, saleor, n8n). Only the dedicated S3 client counts.
118
- registerLibraries({ engine: 's3', category: 'object-storage' }, [
119
- '@aws-sdk/client-s3', 'minio', 'aws-sdk-s3',
120
- ]);
121
- registerLibraries({ engine: 'gcs', category: 'object-storage' }, [
122
- '@google-cloud/storage', 'google-cloud-storage', 'gcloud-storage',
123
- ]);
124
- registerLibraries({ engine: 'azure-blob', category: 'object-storage' }, [
125
- '@azure/storage-blob', 'azure-storage-blob', 'azure-storage',
126
- ]);
127
- registerLibraries({ engine: 'elasticsearch', category: 'search-index' }, [
128
- '@elastic/elasticsearch', 'elasticsearch', 'elasticsearch-py', 'elasticsearch-dsl',
129
- '@opensearch-project/opensearch', 'opensearch-py', 'github.com/elastic/go-elasticsearch',
130
- ]);
131
- registerLibraries({ engine: 'solr', category: 'search-index' }, [
132
- 'solr-client', 'pysolr', 'solr',
133
- ]);
134
- registerLibraries({ engine: 'meilisearch', category: 'search-index' }, [
135
- 'meilisearch', 'meilisearch-python',
136
- ]);
137
- registerLibraries({ engine: 'rabbitmq', category: 'message-queue' }, [
138
- 'amqplib', 'amqp', 'pika', 'bunny', 'github.com/streadway/amqp', 'github.com/rabbitmq/amqp091-go',
139
- ]);
140
- registerLibraries({ engine: 'kafka', category: 'message-queue' }, [
141
- 'kafkajs', 'node-rdkafka', 'kafka-python', 'confluent-kafka', 'github.com/segmentio/kafka-go',
142
- 'sarama', 'github.com/ibm/sarama',
143
- ]);
144
- registerLibraries({ engine: 'sqs', category: 'message-queue' }, [
145
- '@aws-sdk/client-sqs', 'boto3-sqs',
146
- ]);
147
- registerLibraries({ engine: 'nats', category: 'message-queue' }, [
148
- 'nats', 'nats.js', 'nats-py', 'github.com/nats-io/nats.go',
149
- ]);
150
- // Distributed key-value / config stores that hold an app's own coordination
151
- // state (apisix keeps its routes in etcd; vault/apisix can use consul). Keys are
152
- // specific client coordinates — bare `consul`/`etcd` are avoided as too generic.
153
- registerLibraries({ engine: 'etcd', category: 'key-value-store' }, [
154
- 'etcd3', 'python-etcd3', 'go.etcd.io/etcd', 'go.etcd.io/etcd/client/v3',
155
- 'github.com/coreos/etcd', 'coreos-etcd',
156
- ]);
157
- registerLibraries({ engine: 'consul', category: 'key-value-store' }, [
158
- 'python-consul', 'python-consul2', 'github.com/hashicorp/consul/api', 'node-consul',
159
- ]);
160
- // Time-series databases (flagsmith stores analytics in InfluxDB).
161
- registerLibraries({ engine: 'influxdb', category: 'time-series-database' }, [
162
- 'influxdb-client', 'influxdb', '@influxdata/influxdb-client', 'influxdb-python',
163
- // Go clients are imported at their versioned module path.
164
- 'github.com/influxdata/influxdb-client-go', 'github.com/influxdata/influxdb-client-go/v2',
165
- 'github.com/influxdata/influxdb1-client', 'github.com/influxdata/influxdb1-client/v2',
166
- ]);
167
- // Redis-backed job/queue libraries are NOT a separate store — their queue
168
- // lives inside Redis. Mapping them to `redis` folds them into the broker
169
- // (measured: they created a phantom `redis-queue` alongside redis in 8 repos).
170
- // `celery` is omitted because its broker is configurable (Redis or RabbitMQ),
171
- // and the real broker is detected from its own client or a connection URL.
172
- registerLibraries({ engine: 'redis', category: 'key-value-store' }, [
173
- 'bull', 'bullmq', 'bee-queue', 'sidekiq', 'resque', 'rq',
174
- ]);
175
- registerLibraries({ engine: 'sqlite', category: 'embedded-database' }, [
176
- 'sqlite3', 'better-sqlite3', 'sqlite', 'aiosqlite',
177
- 'github.com/mattn/go-sqlite3', 'modernc.org/sqlite', 'github.com/glebarez/sqlite',
178
- 'gorm.io/driver/sqlite',
179
- // .NET
180
- 'microsoft.data.sqlite', 'microsoft.entityframeworkcore.sqlite', 'system.data.sqlite',
181
- ]);
182
- // .NET data-access packages (from .csproj / Directory.Packages.props)
183
- registerLibraries({ engine: 'postgres', category: 'relational-database' }, [
184
- 'npgsql', 'npgsql.entityframeworkcore.postgresql',
185
- ]);
186
- registerLibraries({ engine: 'mysql', category: 'relational-database' }, [
187
- 'mysqlconnector', 'mysql.data', 'pomelo.entityframeworkcore.mysql',
188
- ]);
189
- registerLibraries({ engine: 'sqlserver', category: 'relational-database' }, [
190
- 'microsoft.data.sqlclient', 'system.data.sqlclient',
191
- 'microsoft.entityframeworkcore.sqlserver',
192
- ]);
193
- registerLibraries({ engine: 'oracle', category: 'relational-database' }, [
194
- 'oracle.manageddataaccess', 'oracle.manageddataaccess.core',
195
- ]);
196
- registerLibraries({ engine: 'mongodb', category: 'document-database' }, [
197
- 'mongodb.driver',
198
- ]);
199
- registerLibraries({ engine: 'redis', category: 'key-value-store' }, [
200
- 'stackexchange.redis',
201
- ]);
202
- // ORM/driver packages that pin a specific SQL engine.
203
- registerLibraries({ engine: 'postgres', category: 'relational-database' }, [
204
- 'gorm.io/driver/postgres', 'github.com/uptrace/bun/dialect/pgdialect',
205
- ]);
206
- registerLibraries({ engine: 'mysql', category: 'relational-database' }, [
207
- 'gorm.io/driver/mysql',
208
- ]);
209
- // Elixir Hex drivers (from mix.exs / mix.lock). Each is engine-specific; the
210
- // adapter-agnostic `ecto`/`ecto_sql` are deliberately omitted (they name no
211
- // engine on their own). `ecto_ch`/`clickhouse_ecto` are already in the
212
- // ClickHouse set above.
213
- registerLibraries({ engine: 'postgres', category: 'relational-database' }, ['postgrex']);
214
- registerLibraries({ engine: 'mysql', category: 'relational-database' }, ['myxql']);
215
- registerLibraries({ engine: 'redis', category: 'key-value-store' }, ['redix']);
216
- registerLibraries({ engine: 'mongodb', category: 'document-database' }, ['mongodb_driver']);
217
- registerLibraries({ engine: 'cassandra', category: 'wide-column-database' }, ['xandra']);
218
- // Rust crates (from Cargo.toml). Only engine-specific drivers are named; the
219
- // adapter-agnostic `sqlx`/`diesel`/`sea-orm` are omitted (they name no engine on
220
- // their own — their enabled feature, e.g. "postgres", carries that signal and is
221
- // matched as its own token). The generic `postgres`/`mysql`/`redis`/`mongodb`/
222
- // `clickhouse`/`elasticsearch` crate names already match existing library keys.
223
- registerLibraries({ engine: 'postgres', category: 'relational-database' }, [
224
- 'tokio-postgres', 'deadpool-postgres', 'bb8-postgres', 'sqlx-postgres',
225
- ]);
226
- registerLibraries({ engine: 'mysql', category: 'relational-database' }, ['mysql_async', 'sqlx-mysql']);
227
- registerLibraries({ engine: 'sqlite', category: 'embedded-database' }, ['rusqlite']);
228
- // `fred` is deliberately omitted — it is a common word (author names, comments)
229
- // and the coarse token scan is global across all manifests, so it would mint
230
- // phantom Redis stores. The `-redis`-suffixed crates are unambiguous.
231
- registerLibraries({ engine: 'redis', category: 'key-value-store' }, ['deadpool-redis', 'bb8-redis']);
232
- registerLibraries({ engine: 'kafka', category: 'message-queue' }, ['rdkafka', 'rskafka']);
233
- registerLibraries({ engine: 'rabbitmq', category: 'message-queue' }, ['lapin']);
234
- registerLibraries({ engine: 'cassandra', category: 'wide-column-database' }, ['cdrs-tokio', 'scylla']);
235
- // Container image base name -> engine. Matched against the image before its
236
- // tag and registry path (`postgres:16`, `bitnami/redis`).
237
- // Null-prototype (see LIBRARY_ENGINES): image path segments are arbitrary text.
238
- const IMAGE_ENGINES = Object.assign(Object.create(null), {
239
- postgres: { engine: 'postgres', category: 'relational-database' },
240
- postgresql: { engine: 'postgres', category: 'relational-database' },
241
- postgis: { engine: 'postgres', category: 'relational-database' },
242
- // Postgres distributions/extensions: the store to clone is still Postgres.
243
- pgvector: { engine: 'postgres', category: 'relational-database' },
244
- 'pgvecto-rs': { engine: 'postgres', category: 'relational-database' },
245
- timescaledb: { engine: 'postgres', category: 'relational-database' },
246
- supabase: { engine: 'postgres', category: 'relational-database' },
247
- // Vector databases
248
- weaviate: { engine: 'weaviate', category: 'vector-database' },
249
- qdrant: { engine: 'qdrant', category: 'vector-database' },
250
- milvus: { engine: 'milvus', category: 'vector-database' },
251
- chroma: { engine: 'chroma', category: 'vector-database' },
252
- chromadb: { engine: 'chroma', category: 'vector-database' },
253
- mysql: { engine: 'mysql', category: 'relational-database' },
254
- mariadb: { engine: 'mysql', category: 'relational-database' },
255
- mssql: { engine: 'sqlserver', category: 'relational-database' },
256
- 'mssql-server': { engine: 'sqlserver', category: 'relational-database' },
257
- 'azure-sql-edge': { engine: 'sqlserver', category: 'relational-database' },
258
- mongo: { engine: 'mongodb', category: 'document-database' },
259
- mongodb: { engine: 'mongodb', category: 'document-database' },
260
- couchdb: { engine: 'couchdb', category: 'document-database' },
261
- redis: { engine: 'redis', category: 'key-value-store' },
262
- valkey: { engine: 'redis', category: 'key-value-store' },
263
- clickhouse: { engine: 'clickhouse', category: 'columnar-database' },
264
- 'clickhouse-server': { engine: 'clickhouse', category: 'columnar-database' },
265
- cockroachdb: { engine: 'cockroachdb', category: 'relational-database' },
266
- cockroach: { engine: 'cockroachdb', category: 'relational-database' },
267
- azurite: { engine: 'azure-blob', category: 'object-storage' },
268
- 'servicebus-emulator': { engine: 'azure-servicebus', category: 'message-queue' },
269
- memcached: { engine: 'memcached', category: 'key-value-store' },
270
- cassandra: { engine: 'cassandra', category: 'wide-column-database' },
271
- scylladb: { engine: 'cassandra', category: 'wide-column-database' },
272
- neo4j: { engine: 'neo4j', category: 'graph-database' },
273
- minio: { engine: 's3', category: 'object-storage' },
274
- elasticsearch: { engine: 'elasticsearch', category: 'search-index' },
275
- opensearchproject: { engine: 'elasticsearch', category: 'search-index' },
276
- solr: { engine: 'solr', category: 'search-index' },
277
- meilisearch: { engine: 'meilisearch', category: 'search-index' },
278
- rabbitmq: { engine: 'rabbitmq', category: 'message-queue' },
279
- kafka: { engine: 'kafka', category: 'message-queue' },
280
- nats: { engine: 'nats', category: 'message-queue' },
281
- // Distributed KV / config stores and time-series DBs run as their own service.
282
- etcd: { engine: 'etcd', category: 'key-value-store' },
283
- consul: { engine: 'consul', category: 'key-value-store' },
284
- influxdb: { engine: 'influxdb', category: 'time-series-database' },
285
- });
286
- // URL scheme -> engine, for connection strings in config or source.
287
- // Null-prototype (see LIBRARY_ENGINES): schemes come from arbitrary text.
288
- const URL_SCHEME_ENGINES = Object.assign(Object.create(null), {
289
- 'postgres': { engine: 'postgres', category: 'relational-database' },
290
- 'postgresql': { engine: 'postgres', category: 'relational-database' },
291
- 'mysql': { engine: 'mysql', category: 'relational-database' },
292
- 'mongodb': { engine: 'mongodb', category: 'document-database' },
293
- 'mongodb+srv': { engine: 'mongodb', category: 'document-database' },
294
- 'redis': { engine: 'redis', category: 'key-value-store' },
295
- 'rediss': { engine: 'redis', category: 'key-value-store' },
296
- 'amqp': { engine: 'rabbitmq', category: 'message-queue' },
297
- 'amqps': { engine: 'rabbitmq', category: 'message-queue' },
298
- 'elasticsearch': { engine: 'elasticsearch', category: 'search-index' },
299
- 'clickhouse': { engine: 'clickhouse', category: 'columnar-database' },
300
- 'clickhouses': { engine: 'clickhouse', category: 'columnar-database' },
301
- 'cockroachdb': { engine: 'cockroachdb', category: 'relational-database' },
302
- });
303
- const CONFIDENCE_RANK = {
304
- running: 3, configured: 2, library: 1,
305
- };
306
- function manifestDependencyNames(input) {
307
- const { lowerName, text } = input;
308
- const names = new Set();
309
- if (lowerName === 'package.json') {
310
- try {
311
- const parsed = JSON.parse(text);
312
- for (const field of ['dependencies', 'optionalDependencies', 'peerDependencies']) {
313
- const deps = parsed[field];
314
- if (deps && typeof deps === 'object') {
315
- for (const name of Object.keys(deps))
316
- names.add(name);
317
- }
318
- }
319
- }
320
- catch { /* ignore malformed manifest */ }
321
- return [...names];
322
- }
323
- if (lowerName === 'requirements.txt' || lowerName.startsWith('requirements')) {
324
- for (const line of text.split('\n')) {
325
- const match = line.match(/^\s*([A-Za-z0-9_.-]+)/);
326
- if (match)
327
- names.add(match[1]);
328
- }
329
- return [...names];
330
- }
331
- if (lowerName === 'pyproject.toml' || lowerName === 'go.mod' || lowerName === 'gemfile'
332
- || lowerName === 'composer.json' || lowerName === 'pom.xml'
333
- || lowerName === 'build.gradle' || lowerName === 'build.gradle.kts'
334
- || lowerName === 'mix.exs' || lowerName === 'cargo.toml') {
335
- // Coarse token scan: a store library name appearing anywhere in these
336
- // manifests is a dependency signal. The library map keys are specific
337
- // enough (pg, ioredis, amqplib, cassandra-driver, postgrex, ecto_ch) that
338
- // false matches are rare, and go.mod/pom paths are matched as full module
339
- // coordinates. Elixir deps are `{:postgrex, ...}` atoms, so `{`/`}` split
340
- // too, leaving the bare package name as a token. Only `mix.exs` (direct
341
- // deps) is read, never `mix.lock` — a lockfile is the full transitive tree
342
- // (dev/test deps and their deps), the same reason package-lock.json is
343
- // skipped in favor of package.json. `Cargo.toml` (Rust) is read the same
344
- // way; a store crate's engine-carrying feature (e.g. `features=["postgres"]`)
345
- // matches as its own token, so the adapter-agnostic sqlx/diesel still resolve.
346
- for (const token of text.split(/[\s"',=:(){}[\]]+/)) {
347
- if (token)
348
- names.add(token);
349
- }
350
- return [...names];
351
- }
352
- // .NET project files: PackageReference / PackageVersion Include names. Covers
353
- // .csproj, .fsproj, .vbproj, Directory.Packages.props, Directory.Build.props,
354
- // and packages.config (the whole .NET dependency-manifest surface).
355
- if (lowerName.endsWith('.csproj') || lowerName.endsWith('.fsproj')
356
- || lowerName.endsWith('.vbproj') || lowerName.endsWith('.props')
357
- || lowerName === 'packages.config') {
358
- for (const match of text.matchAll(/<(?:PackageReference|PackageVersion|package)\s+[^>]*\b(?:Include|id)\s*=\s*"([^"]+)"/gi)) {
359
- names.add(match[1]);
360
- }
361
- return [...names];
362
- }
363
- return [];
364
- }
365
- // CI pipeline config paths. A `services:` container declared in a CI workflow
366
- // (GitHub Actions, GitLab, CircleCI, ...) is ephemeral test infrastructure, not
367
- // the application's production topology — a Postgres spun up to run the test
368
- // suite is not a store the app ships with. Scanning these for `image:` invents
369
- // running stores (measured: twenty's ClickHouse came from .github/workflows,
370
- // not its compose). Matched on the normalized repo-relative path, not content.
371
- function isCiConfigPath(path) {
372
- const p = path.toLowerCase().replace(/\\/g, '/');
373
- return p.includes('.github/workflows/')
374
- || p.includes('.gitea/workflows/')
375
- || p.includes('.forgejo/workflows/')
376
- || p.includes('.circleci/')
377
- || p.includes('.buildkite/')
378
- || /(^|\/)\.gitlab-ci\.ya?ml$/.test(p)
379
- || /(^|\/)\.drone\.ya?ml$/.test(p)
380
- || /(^|\/)\.woodpecker\.ya?ml$/.test(p)
381
- || p.includes('/.woodpecker/')
382
- || /(^|\/)bitbucket-pipelines\.ya?ml$/.test(p)
383
- || /(^|\/)azure-pipelines\.ya?ml$/.test(p);
384
- }
385
- function serviceImages(input) {
386
- const { lowerName, text } = input;
387
- const isCompose = /(^|\/)(docker-)?compose\.ya?ml$/.test(lowerName)
388
- || lowerName === 'compose.yml' || lowerName === 'compose.yaml'
389
- || lowerName === 'docker-compose.yml' || lowerName === 'docker-compose.yaml';
390
- const isYaml = lowerName.endsWith('.yml') || lowerName.endsWith('.yaml');
391
- if (!isCompose && !isYaml)
392
- return [];
393
- const images = [];
394
- for (const line of text.split('\n')) {
395
- const match = line.match(/^\s*(?:-\s*)?image\s*:\s*["']?([^"'\s]+)["']?/);
396
- if (match)
397
- images.push(match[1]);
398
- }
399
- return images;
400
- }
401
- function connectionUrlSchemes(input) {
402
- const schemes = [];
403
- for (const match of input.text.matchAll(/\b([a-z][a-z0-9+.-]*)\s*:\/\//gi)) {
404
- schemes.push(match[1].toLowerCase());
405
- }
406
- return schemes;
407
- }
408
- // High-signal config declarations that name a store without an image, a client
409
- // library, or a URL. Two whole ecosystems configure their stores this way —
410
- // Python/Django (`DATABASES … ENGINE`), and PHP (built-in PDO/mysqli named only
411
- // in an .ini/.php config) — plus the conventional `<ENGINE>_HOST` env keys. On
412
- // the survey these were the dominant recall gap (zulip's four stores, matomo's
413
- // database). Kept deliberately specific to avoid matching prose or an example.
414
- const CONFIG_ENGINE_PATTERNS = [
415
- // Django database backends: `django.db.backends.postgresql`
416
- { re: /django\.db\.backends\.postgresql/i, profile: { engine: 'postgres', category: 'relational-database' } },
417
- { re: /django\.db\.backends\.mysql/i, profile: { engine: 'mysql', category: 'relational-database' } },
418
- { re: /django\.db\.backends\.oracle/i, profile: { engine: 'oracle', category: 'relational-database' } },
419
- { re: /django\.db\.backends\.sqlite3/i, profile: { engine: 'sqlite', category: 'embedded-database' } },
420
- // Django cache backends
421
- { re: /django(?:\.core\.cache\.backends\.memcached|_bmemcached|\.core\.cache\.backends\.pymemcache)/i, profile: { engine: 'memcached', category: 'key-value-store' } },
422
- { re: /(?:django_redis|django\.core\.cache\.backends\.redis)/i, profile: { engine: 'redis', category: 'key-value-store' } },
423
- // PHP built-in DB drivers named only in config (matomo, many PHP apps)
424
- { re: /\b(?:adapter\s*=\s*)?PDO\\?_?MYSQL\b/i, profile: { engine: 'mysql', category: 'relational-database' } },
425
- { re: /\b(?:adapter\s*=\s*)?PDO\\?_?PGSQL\b/i, profile: { engine: 'postgres', category: 'relational-database' } },
426
- // Laravel .env: DB_CONNECTION names the built-in PDO driver (firefly, koel)
427
- { re: /\bDB_CONNECTION\s*=\s*["']?(?:mysql|mariadb)\b/i, profile: { engine: 'mysql', category: 'relational-database' } },
428
- { re: /\bDB_CONNECTION\s*=\s*["']?(?:pgsql|postgres)\b/i, profile: { engine: 'postgres', category: 'relational-database' } },
429
- { re: /\bDB_CONNECTION\s*=\s*["']?sqlite\b/i, profile: { engine: 'sqlite', category: 'embedded-database' } },
430
- { re: /\bDB_CONNECTION\s*=\s*["']?sqlsrv\b/i, profile: { engine: 'sqlserver', category: 'relational-database' } },
431
- // Rails config/database.yml: adapter names the driver (chatwoot, many Rails)
432
- { re: /\badapter\s*:\s*postgresql\b/i, profile: { engine: 'postgres', category: 'relational-database' } },
433
- { re: /\badapter\s*:\s*mysql2?\b/i, profile: { engine: 'mysql', category: 'relational-database' } },
434
- { re: /\badapter\s*:\s*sqlite3\b/i, profile: { engine: 'sqlite', category: 'embedded-database' } },
435
- { re: /\badapter\s*:\s*sqlserver\b/i, profile: { engine: 'sqlserver', category: 'relational-database' } },
436
- // Conventional <ENGINE>_HOST / _DATABASE env keys for the SQL engines too
437
- { re: /\bPOSTGRES(?:QL)?_(?:HOST|DB|DATABASE)\b/i, profile: { engine: 'postgres', category: 'relational-database' } },
438
- { re: /\bPG(?:HOST|DATABASE)\b/, profile: { engine: 'postgres', category: 'relational-database' } },
439
- { re: /\bMYSQL_(?:HOST|DATABASE|DB)\b/i, profile: { engine: 'mysql', category: 'relational-database' } },
440
- // Conventional <ENGINE>_HOST / _LOCATION / _SERVERS / _CONTACT_POINTS env keys
441
- { re: /\bREDIS_(?:HOST|HOSTS|SERVER|URL)\b/i, profile: { engine: 'redis', category: 'key-value-store' } },
442
- { re: /\bMEMCACHED_(?:HOST|HOSTS|LOCATION|SERVERS)\b/i, profile: { engine: 'memcached', category: 'key-value-store' } },
443
- { re: /\bRABBITMQ_(?:HOST|HOSTS|URL)\b/i, profile: { engine: 'rabbitmq', category: 'message-queue' } },
444
- { re: /\bKAFKA_(?:BROKERS|BOOTSTRAP|BOOTSTRAP_SERVERS|HOSTS?|SERVERS)\b/i, profile: { engine: 'kafka', category: 'message-queue' } },
445
- { re: /\bCLICKHOUSE_(?:HOST|URL|DSN|ADDR)\b/i, profile: { engine: 'clickhouse', category: 'columnar-database' } },
446
- { re: /\bCASSANDRA_(?:HOST|HOSTS|CONTACT_POINTS|SEEDS)\b/i, profile: { engine: 'cassandra', category: 'wide-column-database' } },
447
- { re: /\bMONGO(?:DB)?_(?:HOST|URL|URI)\b/i, profile: { engine: 'mongodb', category: 'document-database' } },
448
- { re: /\bELASTICSEARCH_(?:HOST|HOSTS|URL)\b/i, profile: { engine: 'elasticsearch', category: 'search-index' } },
449
- { re: /\bETCD_(?:HOST|HOSTS|ENDPOINT|ENDPOINTS)\b/i, profile: { engine: 'etcd', category: 'key-value-store' } },
450
- { re: /\bINFLUX(?:DB)?_(?:HOST|URL|BUCKET|ORG)\b/i, profile: { engine: 'influxdb', category: 'time-series-database' } },
451
- { re: /\bCONSUL_(?:HTTP_ADDR|HOST|URL)\b/i, profile: { engine: 'consul', category: 'key-value-store' } },
452
- ];
453
- function configEngineSignals(input) {
454
- const found = [];
455
- for (const { re, profile } of CONFIG_ENGINE_PATTERNS) {
456
- if (re.test(input.text))
457
- found.push(profile);
458
- }
459
- return found;
460
- }
461
- function imageEngineFor(image) {
462
- // Drop the digest, then test each path segment against the engine map, last
463
- // segment first. `bitnami/postgresql:16` matches on `postgresql`;
464
- // `mcr.microsoft.com/mssql/server` matches on the `mssql` segment (the engine
465
- // is not always the final segment behind a registry path).
466
- const segments = image.split('@')[0].split('/').map((segment) => segment.toLowerCase());
467
- // Strip the tag from the final segment ONLY — a colon in an earlier segment is
468
- // a registry port (`localhost:5000/postgres:16`), not a tag separator.
469
- segments[segments.length - 1] = segments[segments.length - 1].split(':')[0];
470
- for (let index = segments.length - 1; index >= 0; index -= 1) {
471
- const profile = IMAGE_ENGINES[segments[index]];
472
- if (profile)
473
- return profile;
474
- }
475
- return undefined;
476
- }
477
- /**
478
- * Fold a stream of manifest/compose/config files into the distinct data stores
479
- * the application uses. `inputs` is every scanned text file with its lowercased
480
- * basename; the detector reads only the manifests and configs it recognizes.
481
- */
482
- export function detectDataStores(inputs) {
483
- const byEngine = new Map();
484
- const upsert = (profile, confidence, signal) => {
485
- const existing = byEngine.get(profile.engine);
486
- if (existing) {
487
- existing.signals.push(signal);
488
- if (CONFIDENCE_RANK[confidence] > CONFIDENCE_RANK[existing.confidence]) {
489
- existing.confidence = confidence;
490
- }
491
- return;
492
- }
493
- byEngine.set(profile.engine, {
494
- engine: profile.engine,
495
- category: profile.category,
496
- clone_strategy: CLONE_STRATEGY_BY_CATEGORY[profile.category],
497
- confidence,
498
- signals: [signal],
499
- });
500
- };
501
- for (const input of inputs) {
502
- // A CI pipeline file is test infrastructure end to end: its service images,
503
- // its `POSTGRES_HOST`/`REDIS_URL` env, and its connection strings all describe
504
- // the test harness, not the app's production stores. Skip it for every signal
505
- // (it is never a dependency manifest, so nothing of value is lost).
506
- if (isCiConfigPath(input.path))
507
- continue;
508
- for (const image of serviceImages(input)) {
509
- const profile = imageEngineFor(image);
510
- if (profile)
511
- upsert(profile, 'running', { kind: 'service-image', value: image, path: input.path });
512
- }
513
- for (const dependency of manifestDependencyNames(input)) {
514
- const profile = LIBRARY_ENGINES[dependency.toLowerCase()];
515
- if (profile)
516
- upsert(profile, 'library', { kind: 'client-library', value: dependency, path: input.path });
517
- }
518
- for (const scheme of connectionUrlSchemes(input)) {
519
- const profile = URL_SCHEME_ENGINES[scheme];
520
- if (profile)
521
- upsert(profile, 'configured', { kind: 'connection-url', value: `${scheme}://`, path: input.path });
522
- }
523
- for (const profile of configEngineSignals(input)) {
524
- upsert(profile, 'configured', { kind: 'config-declaration', value: profile.engine, path: input.path });
525
- }
526
- }
527
- return [...byEngine.values()]
528
- .map((store) => ({
529
- ...store,
530
- // Cap and stabilize evidence for a deterministic, bounded record.
531
- signals: store.signals
532
- .slice()
533
- .sort((left, right) => left.kind.localeCompare(right.kind) ||
534
- left.path.localeCompare(right.path) ||
535
- left.value.localeCompare(right.value))
536
- .slice(0, 24),
537
- }))
538
- .sort((left, right) => left.engine.localeCompare(right.engine));
539
- }