@haystackeditor/cli 0.25.1 → 0.26.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (97) hide show
  1. package/README.md +39 -455
  2. package/dist/capture/app-config.js +25 -1
  3. package/dist/commands/capture-brief.js +41 -32
  4. package/dist/commands/feedback.js +66 -0
  5. package/dist/commands/init-telemetry.js +53 -7
  6. package/dist/commands/init.js +7 -2
  7. package/dist/commands/lockfile-pin.js +307 -0
  8. package/dist/commands/verify.js +68 -13
  9. package/dist/index.js +27 -923
  10. package/dist/schema.js +4 -10
  11. package/dist/utils/haystack-api.js +0 -36
  12. package/package.json +1 -5
  13. package/schemas/feedback.v1.json +13 -0
  14. package/schemas/pre-verify.v2.json +240 -0
  15. package/schemas/verify-raw.v1.json +1132 -0
  16. package/schemas/verify.v2.json +655 -0
  17. package/dist/assets/hooks/agent-context/detect.ts +0 -316
  18. package/dist/assets/hooks/agent-context/format.ts +0 -100
  19. package/dist/assets/hooks/agent-context/index.ts +0 -41
  20. package/dist/assets/hooks/agent-context/parsers/claude.ts +0 -262
  21. package/dist/assets/hooks/agent-context/parsers/codex.ts +0 -416
  22. package/dist/assets/hooks/agent-context/parsers/gemini.ts +0 -155
  23. package/dist/assets/hooks/agent-context/parsers/opencode.ts +0 -174
  24. package/dist/assets/hooks/agent-context/tsconfig.json +0 -14
  25. package/dist/assets/hooks/agent-context/types.ts +0 -58
  26. package/dist/assets/hooks/llm-rules-template.md +0 -59
  27. package/dist/assets/hooks/package-lock.json +0 -598
  28. package/dist/assets/hooks/package.json +0 -12
  29. package/dist/assets/hooks/scripts/commit-msg.sh +0 -5
  30. package/dist/assets/hooks/scripts/post-commit.sh +0 -5
  31. package/dist/assets/hooks/scripts/pre-commit.sh +0 -175
  32. package/dist/assets/hooks/scripts/pre-push.sh +0 -25
  33. package/dist/assets/hooks/scripts/prepare-commit-msg.sh +0 -5
  34. package/dist/assets/hooks/truncation-checker/ast-analyzer.ts +0 -528
  35. package/dist/assets/hooks/truncation-checker/index.ts +0 -595
  36. package/dist/assets/hooks/truncation-checker/tsconfig.json +0 -13
  37. package/dist/assets/skills/map-cloud-verifier-universe/SKILL.md +0 -2051
  38. package/dist/assets/skills/map-cloud-verifier-universe/agents/openai.yaml +0 -4
  39. package/dist/assets/skills/map-cloud-verifier-universe/references/output-contract.md +0 -3411
  40. package/dist/assets/skills/map-your-system.md +0 -143
  41. package/dist/assets/skills/submit.md +0 -200
  42. package/dist/commands/ask.js +0 -20
  43. package/dist/commands/cloud-verifier-behaviors.js +0 -218
  44. package/dist/commands/cloud-verifier-data-store-census.js +0 -539
  45. package/dist/commands/cloud-verifier-data-store-drift.js +0 -158
  46. package/dist/commands/cloud-verifier-identity-census.js +0 -4060
  47. package/dist/commands/cloud-verifier-materialization.js +0 -704
  48. package/dist/commands/cloud-verifier-pascal-selector-census.js +0 -1382
  49. package/dist/commands/cloud-verifier-python-manifest-selector-census.js +0 -2015
  50. package/dist/commands/cloud-verifier-specialized-operational-census.js +0 -11432
  51. package/dist/commands/cloud-verifier-universe.js +0 -10178
  52. package/dist/commands/config.js +0 -549
  53. package/dist/commands/design-verify.js +0 -311
  54. package/dist/commands/dismiss.js +0 -159
  55. package/dist/commands/hooks.js +0 -226
  56. package/dist/commands/inbox.js +0 -137
  57. package/dist/commands/mcp.js +0 -201
  58. package/dist/commands/policy.js +0 -371
  59. package/dist/commands/pr-status.js +0 -207
  60. package/dist/commands/pr.js +0 -105
  61. package/dist/commands/prepare-universe-review.js +0 -1092
  62. package/dist/commands/production-source-deny-policy.js +0 -100
  63. package/dist/commands/request-review.js +0 -74
  64. package/dist/commands/review.js +0 -191
  65. package/dist/commands/rules.js +0 -98
  66. package/dist/commands/scaffold-provisional-universe.js +0 -806
  67. package/dist/commands/setup.js +0 -1170
  68. package/dist/commands/skills.js +0 -447
  69. package/dist/commands/submit.js +0 -745
  70. package/dist/commands/system-map.js +0 -228
  71. package/dist/commands/triage.js +0 -598
  72. package/dist/commands/webhooks.js +0 -241
  73. package/dist/states.js +0 -46
  74. package/dist/tools/detect.js +0 -832
  75. package/dist/triage/astra.js +0 -202
  76. package/dist/triage/prompts.js +0 -188
  77. package/dist/triage/runner.js +0 -200
  78. package/dist/triage/types.js +0 -7
  79. package/dist/utils/action-output.js +0 -26
  80. package/dist/utils/analysis-api.js +0 -416
  81. package/dist/utils/design-verifier-api.js +0 -294
  82. package/dist/utils/design-verifier-history.js +0 -79
  83. package/dist/utils/design-verifier-result.js +0 -424
  84. package/dist/utils/github-api.js +0 -324
  85. package/dist/utils/pending-state.js +0 -86
  86. package/dist/utils/pr-ref.js +0 -56
  87. package/dist/utils/prompter.js +0 -328
  88. package/schemas/action.v1.json +0 -22
  89. package/schemas/ask.v1.json +0 -40
  90. package/schemas/inbox.v1.json +0 -27
  91. package/schemas/pr-status.v1.json +0 -61
  92. package/schemas/pr.v1.json +0 -97
  93. package/schemas/pr.v3.json +0 -45
  94. package/schemas/setup.v1.json +0 -75
  95. package/schemas/submit.v1.json +0 -90
  96. package/schemas/triage.v1.json +0 -103
  97. package/schemas/triage.v2.json +0 -64
@@ -1,539 +0,0 @@
1
- /**
2
- * Data-store census.
3
- *
4
- * The one job the Cloud Verifier needs from a repository map: what data stores
5
- * does this application use, so we can clone each one into a replica. Not
6
- * processes, not API calls, not code structure — the databases, caches, object
7
- * stores, search indexes, and queues that hold state a test must run against.
8
- *
9
- * Four source signals, strongest first:
10
- * 1. a service image in a compose/k8s manifest (`image: postgres:16`) — a
11
- * store instance the app is wired to run beside, the surest signal;
12
- * 2. a connection URL scheme or conventional env var (`postgres://`,
13
- * `REDIS_URL`) — an instance named in config;
14
- * 3. a config declaration that names a store without an image, library, or URL
15
- * (Django `ENGINE`, PHP PDO, Laravel `DB_CONNECTION`, Rails `adapter:`);
16
- * 4. a client library in a dependency manifest (`pg`, `redis`, `mongoose`) —
17
- * the app talks to that engine.
18
- *
19
- * Signals are folded per engine into one data store, with the evidence, a
20
- * confidence, and how to reproduce it. Everything here is a general convention,
21
- * never a repository-specific constant.
22
- */
23
- // How a replica of this category is made. Durable stores of record are copied
24
- // from a snapshot; caches and search indexes are usually derived and can be
25
- // rebuilt; queues are drained to empty.
26
- const CLONE_STRATEGY_BY_CATEGORY = {
27
- 'relational-database': 'snapshot-restore',
28
- 'document-database': 'snapshot-restore',
29
- 'key-value-store': 'snapshot-restore',
30
- 'wide-column-database': 'snapshot-restore',
31
- 'columnar-database': 'snapshot-restore',
32
- 'graph-database': 'snapshot-restore',
33
- 'vector-database': 'snapshot-restore',
34
- 'object-storage': 'copy-bucket-snapshot',
35
- 'search-index': 'rebuild-from-source',
36
- 'message-queue': 'reset-empty',
37
- 'time-series-database': 'snapshot-restore',
38
- 'embedded-database': 'copy-file',
39
- };
40
- // Client library (as written in a dependency manifest) -> engine. Covers the
41
- // common stores across Node, Python, Go, Ruby, Java, PHP. Keys are matched
42
- // case-insensitively against exact dependency names.
43
- // Null-prototype: keys are matched against arbitrary dependency tokens from a
44
- // manifest, so a token like `constructor`/`toString`/`hasOwnProperty` must NOT
45
- // resolve to an inherited Object.prototype member (that returned a truthy
46
- // non-profile and created a store with an undefined engine — crashed on a repo
47
- // whose manifest contained such a token).
48
- const LIBRARY_ENGINES = Object.create(null);
49
- function registerLibraries(profile, names) {
50
- for (const name of names)
51
- LIBRARY_ENGINES[name.toLowerCase()] = profile;
52
- }
53
- registerLibraries({ engine: 'postgres', category: 'relational-database' }, [
54
- 'pg', 'pg-promise', 'postgres', 'node-postgres', 'psycopg2', 'psycopg2-binary',
55
- 'psycopg', 'asyncpg', 'lib/pq', 'github.com/lib/pq', 'github.com/jackc/pgx',
56
- 'github.com/jackc/pgx/v5', 'pg gem', 'postgresql', 'org.postgresql:postgresql',
57
- ]);
58
- registerLibraries({ engine: 'mysql', category: 'relational-database' }, [
59
- 'mysql', 'mysql2', 'mariadb', 'pymysql', 'mysqlclient', 'aiomysql',
60
- 'github.com/go-sql-driver/mysql', 'mysql-connector-python', 'mysql:mysql-connector-java',
61
- ]);
62
- registerLibraries({ engine: 'sqlserver', category: 'relational-database' }, [
63
- 'mssql', 'tedious', 'pyodbc', 'pymssql', 'github.com/denisenkom/go-mssqldb',
64
- ]);
65
- registerLibraries({ engine: 'oracle', category: 'relational-database' }, [
66
- 'oracledb', 'cx_oracle', 'cx-oracle',
67
- ]);
68
- registerLibraries({ engine: 'mongodb', category: 'document-database' }, [
69
- 'mongodb', 'mongoose', 'pymongo', 'motor', 'mongoengine',
70
- 'go.mongodb.org/mongo-driver', 'mongo', 'mongoid',
71
- ]);
72
- registerLibraries({ engine: 'couchdb', category: 'document-database' }, [
73
- 'nano', 'couchdb', 'pycouchdb', 'cloudant',
74
- ]);
75
- registerLibraries({ engine: 'redis', category: 'key-value-store' }, [
76
- 'redis', 'ioredis', 'redis-py', 'connect-redis', 'go-redis', 'github.com/redis/go-redis',
77
- 'github.com/go-redis/redis', 'hiredis', 'aioredis', 'redis-rb', 'predis/predis', 'jedis',
78
- ]);
79
- registerLibraries({ engine: 'memcached', category: 'key-value-store' }, [
80
- 'memcached', 'memjs', 'pymemcache', 'python-memcached', 'dalli',
81
- ]);
82
- registerLibraries({ engine: 'dynamodb', category: 'key-value-store' }, [
83
- '@aws-sdk/client-dynamodb', 'dynamoose',
84
- ]);
85
- registerLibraries({ engine: 'cassandra', category: 'wide-column-database' }, [
86
- 'cassandra-driver', 'cassandra', 'scylla-driver', 'datastax',
87
- ]);
88
- registerLibraries({ engine: 'clickhouse', category: 'columnar-database' }, [
89
- '@clickhouse/client', '@clickhouse/client-web', 'clickhouse-driver',
90
- 'clickhouse-connect', 'clickhouse', 'ecto_ch', 'clickhouse_ecto',
91
- 'github.com/clickhouse/clickhouse-go', 'clickhouse-sqlalchemy',
92
- ]);
93
- registerLibraries({ engine: 'cockroachdb', category: 'relational-database' }, [
94
- 'cockroachdb', 'sqlalchemy-cockroachdb',
95
- ]);
96
- registerLibraries({ engine: 'neo4j', category: 'graph-database' }, [
97
- 'neo4j-driver', 'neo4j', 'py2neo', 'neo4j-python-driver',
98
- ]);
99
- registerLibraries({ engine: 'weaviate', category: 'vector-database' }, [
100
- 'weaviate-client', 'weaviate-ts-client', '@weaviate/client', 'weaviate',
101
- ]);
102
- registerLibraries({ engine: 'qdrant', category: 'vector-database' }, [
103
- 'qdrant-client', '@qdrant/js-client-rest', 'qdrant',
104
- ]);
105
- registerLibraries({ engine: 'milvus', category: 'vector-database' }, [
106
- 'pymilvus', '@zilliz/milvus2-sdk-node', 'milvus',
107
- ]);
108
- registerLibraries({ engine: 'pinecone', category: 'vector-database' }, [
109
- 'pinecone-client', '@pinecone-database/pinecone', 'pinecone',
110
- ]);
111
- registerLibraries({ engine: 'chroma', category: 'vector-database' }, [
112
- 'chromadb', 'chromadb-client',
113
- ]);
114
- // Object-storage clients must be the specific S3 client, not a monolithic
115
- // cloud SDK: bare `aws-sdk`/`boto3`/`aws-sdk-go` are used for SES, SNS, SQS,
116
- // and dozens of non-storage services, so mapping them to S3 fabricates stores
117
- // (measured on medusa, saleor, n8n). Only the dedicated S3 client counts.
118
- registerLibraries({ engine: 's3', category: 'object-storage' }, [
119
- '@aws-sdk/client-s3', 'minio', 'aws-sdk-s3',
120
- ]);
121
- registerLibraries({ engine: 'gcs', category: 'object-storage' }, [
122
- '@google-cloud/storage', 'google-cloud-storage', 'gcloud-storage',
123
- ]);
124
- registerLibraries({ engine: 'azure-blob', category: 'object-storage' }, [
125
- '@azure/storage-blob', 'azure-storage-blob', 'azure-storage',
126
- ]);
127
- registerLibraries({ engine: 'elasticsearch', category: 'search-index' }, [
128
- '@elastic/elasticsearch', 'elasticsearch', 'elasticsearch-py', 'elasticsearch-dsl',
129
- '@opensearch-project/opensearch', 'opensearch-py', 'github.com/elastic/go-elasticsearch',
130
- ]);
131
- registerLibraries({ engine: 'solr', category: 'search-index' }, [
132
- 'solr-client', 'pysolr', 'solr',
133
- ]);
134
- registerLibraries({ engine: 'meilisearch', category: 'search-index' }, [
135
- 'meilisearch', 'meilisearch-python',
136
- ]);
137
- registerLibraries({ engine: 'rabbitmq', category: 'message-queue' }, [
138
- 'amqplib', 'amqp', 'pika', 'bunny', 'github.com/streadway/amqp', 'github.com/rabbitmq/amqp091-go',
139
- ]);
140
- registerLibraries({ engine: 'kafka', category: 'message-queue' }, [
141
- 'kafkajs', 'node-rdkafka', 'kafka-python', 'confluent-kafka', 'github.com/segmentio/kafka-go',
142
- 'sarama', 'github.com/ibm/sarama',
143
- ]);
144
- registerLibraries({ engine: 'sqs', category: 'message-queue' }, [
145
- '@aws-sdk/client-sqs', 'boto3-sqs',
146
- ]);
147
- registerLibraries({ engine: 'nats', category: 'message-queue' }, [
148
- 'nats', 'nats.js', 'nats-py', 'github.com/nats-io/nats.go',
149
- ]);
150
- // Distributed key-value / config stores that hold an app's own coordination
151
- // state (apisix keeps its routes in etcd; vault/apisix can use consul). Keys are
152
- // specific client coordinates — bare `consul`/`etcd` are avoided as too generic.
153
- registerLibraries({ engine: 'etcd', category: 'key-value-store' }, [
154
- 'etcd3', 'python-etcd3', 'go.etcd.io/etcd', 'go.etcd.io/etcd/client/v3',
155
- 'github.com/coreos/etcd', 'coreos-etcd',
156
- ]);
157
- registerLibraries({ engine: 'consul', category: 'key-value-store' }, [
158
- 'python-consul', 'python-consul2', 'github.com/hashicorp/consul/api', 'node-consul',
159
- ]);
160
- // Time-series databases (flagsmith stores analytics in InfluxDB).
161
- registerLibraries({ engine: 'influxdb', category: 'time-series-database' }, [
162
- 'influxdb-client', 'influxdb', '@influxdata/influxdb-client', 'influxdb-python',
163
- // Go clients are imported at their versioned module path.
164
- 'github.com/influxdata/influxdb-client-go', 'github.com/influxdata/influxdb-client-go/v2',
165
- 'github.com/influxdata/influxdb1-client', 'github.com/influxdata/influxdb1-client/v2',
166
- ]);
167
- // Redis-backed job/queue libraries are NOT a separate store — their queue
168
- // lives inside Redis. Mapping them to `redis` folds them into the broker
169
- // (measured: they created a phantom `redis-queue` alongside redis in 8 repos).
170
- // `celery` is omitted because its broker is configurable (Redis or RabbitMQ),
171
- // and the real broker is detected from its own client or a connection URL.
172
- registerLibraries({ engine: 'redis', category: 'key-value-store' }, [
173
- 'bull', 'bullmq', 'bee-queue', 'sidekiq', 'resque', 'rq',
174
- ]);
175
- registerLibraries({ engine: 'sqlite', category: 'embedded-database' }, [
176
- 'sqlite3', 'better-sqlite3', 'sqlite', 'aiosqlite',
177
- 'github.com/mattn/go-sqlite3', 'modernc.org/sqlite', 'github.com/glebarez/sqlite',
178
- 'gorm.io/driver/sqlite',
179
- // .NET
180
- 'microsoft.data.sqlite', 'microsoft.entityframeworkcore.sqlite', 'system.data.sqlite',
181
- ]);
182
- // .NET data-access packages (from .csproj / Directory.Packages.props)
183
- registerLibraries({ engine: 'postgres', category: 'relational-database' }, [
184
- 'npgsql', 'npgsql.entityframeworkcore.postgresql',
185
- ]);
186
- registerLibraries({ engine: 'mysql', category: 'relational-database' }, [
187
- 'mysqlconnector', 'mysql.data', 'pomelo.entityframeworkcore.mysql',
188
- ]);
189
- registerLibraries({ engine: 'sqlserver', category: 'relational-database' }, [
190
- 'microsoft.data.sqlclient', 'system.data.sqlclient',
191
- 'microsoft.entityframeworkcore.sqlserver',
192
- ]);
193
- registerLibraries({ engine: 'oracle', category: 'relational-database' }, [
194
- 'oracle.manageddataaccess', 'oracle.manageddataaccess.core',
195
- ]);
196
- registerLibraries({ engine: 'mongodb', category: 'document-database' }, [
197
- 'mongodb.driver',
198
- ]);
199
- registerLibraries({ engine: 'redis', category: 'key-value-store' }, [
200
- 'stackexchange.redis',
201
- ]);
202
- // ORM/driver packages that pin a specific SQL engine.
203
- registerLibraries({ engine: 'postgres', category: 'relational-database' }, [
204
- 'gorm.io/driver/postgres', 'github.com/uptrace/bun/dialect/pgdialect',
205
- ]);
206
- registerLibraries({ engine: 'mysql', category: 'relational-database' }, [
207
- 'gorm.io/driver/mysql',
208
- ]);
209
- // Elixir Hex drivers (from mix.exs / mix.lock). Each is engine-specific; the
210
- // adapter-agnostic `ecto`/`ecto_sql` are deliberately omitted (they name no
211
- // engine on their own). `ecto_ch`/`clickhouse_ecto` are already in the
212
- // ClickHouse set above.
213
- registerLibraries({ engine: 'postgres', category: 'relational-database' }, ['postgrex']);
214
- registerLibraries({ engine: 'mysql', category: 'relational-database' }, ['myxql']);
215
- registerLibraries({ engine: 'redis', category: 'key-value-store' }, ['redix']);
216
- registerLibraries({ engine: 'mongodb', category: 'document-database' }, ['mongodb_driver']);
217
- registerLibraries({ engine: 'cassandra', category: 'wide-column-database' }, ['xandra']);
218
- // Rust crates (from Cargo.toml). Only engine-specific drivers are named; the
219
- // adapter-agnostic `sqlx`/`diesel`/`sea-orm` are omitted (they name no engine on
220
- // their own — their enabled feature, e.g. "postgres", carries that signal and is
221
- // matched as its own token). The generic `postgres`/`mysql`/`redis`/`mongodb`/
222
- // `clickhouse`/`elasticsearch` crate names already match existing library keys.
223
- registerLibraries({ engine: 'postgres', category: 'relational-database' }, [
224
- 'tokio-postgres', 'deadpool-postgres', 'bb8-postgres', 'sqlx-postgres',
225
- ]);
226
- registerLibraries({ engine: 'mysql', category: 'relational-database' }, ['mysql_async', 'sqlx-mysql']);
227
- registerLibraries({ engine: 'sqlite', category: 'embedded-database' }, ['rusqlite']);
228
- // `fred` is deliberately omitted — it is a common word (author names, comments)
229
- // and the coarse token scan is global across all manifests, so it would mint
230
- // phantom Redis stores. The `-redis`-suffixed crates are unambiguous.
231
- registerLibraries({ engine: 'redis', category: 'key-value-store' }, ['deadpool-redis', 'bb8-redis']);
232
- registerLibraries({ engine: 'kafka', category: 'message-queue' }, ['rdkafka', 'rskafka']);
233
- registerLibraries({ engine: 'rabbitmq', category: 'message-queue' }, ['lapin']);
234
- registerLibraries({ engine: 'cassandra', category: 'wide-column-database' }, ['cdrs-tokio', 'scylla']);
235
- // Container image base name -> engine. Matched against the image before its
236
- // tag and registry path (`postgres:16`, `bitnami/redis`).
237
- // Null-prototype (see LIBRARY_ENGINES): image path segments are arbitrary text.
238
- const IMAGE_ENGINES = Object.assign(Object.create(null), {
239
- postgres: { engine: 'postgres', category: 'relational-database' },
240
- postgresql: { engine: 'postgres', category: 'relational-database' },
241
- postgis: { engine: 'postgres', category: 'relational-database' },
242
- // Postgres distributions/extensions: the store to clone is still Postgres.
243
- pgvector: { engine: 'postgres', category: 'relational-database' },
244
- 'pgvecto-rs': { engine: 'postgres', category: 'relational-database' },
245
- timescaledb: { engine: 'postgres', category: 'relational-database' },
246
- supabase: { engine: 'postgres', category: 'relational-database' },
247
- // Vector databases
248
- weaviate: { engine: 'weaviate', category: 'vector-database' },
249
- qdrant: { engine: 'qdrant', category: 'vector-database' },
250
- milvus: { engine: 'milvus', category: 'vector-database' },
251
- chroma: { engine: 'chroma', category: 'vector-database' },
252
- chromadb: { engine: 'chroma', category: 'vector-database' },
253
- mysql: { engine: 'mysql', category: 'relational-database' },
254
- mariadb: { engine: 'mysql', category: 'relational-database' },
255
- mssql: { engine: 'sqlserver', category: 'relational-database' },
256
- 'mssql-server': { engine: 'sqlserver', category: 'relational-database' },
257
- 'azure-sql-edge': { engine: 'sqlserver', category: 'relational-database' },
258
- mongo: { engine: 'mongodb', category: 'document-database' },
259
- mongodb: { engine: 'mongodb', category: 'document-database' },
260
- couchdb: { engine: 'couchdb', category: 'document-database' },
261
- redis: { engine: 'redis', category: 'key-value-store' },
262
- valkey: { engine: 'redis', category: 'key-value-store' },
263
- clickhouse: { engine: 'clickhouse', category: 'columnar-database' },
264
- 'clickhouse-server': { engine: 'clickhouse', category: 'columnar-database' },
265
- cockroachdb: { engine: 'cockroachdb', category: 'relational-database' },
266
- cockroach: { engine: 'cockroachdb', category: 'relational-database' },
267
- azurite: { engine: 'azure-blob', category: 'object-storage' },
268
- 'servicebus-emulator': { engine: 'azure-servicebus', category: 'message-queue' },
269
- memcached: { engine: 'memcached', category: 'key-value-store' },
270
- cassandra: { engine: 'cassandra', category: 'wide-column-database' },
271
- scylladb: { engine: 'cassandra', category: 'wide-column-database' },
272
- neo4j: { engine: 'neo4j', category: 'graph-database' },
273
- minio: { engine: 's3', category: 'object-storage' },
274
- elasticsearch: { engine: 'elasticsearch', category: 'search-index' },
275
- opensearchproject: { engine: 'elasticsearch', category: 'search-index' },
276
- solr: { engine: 'solr', category: 'search-index' },
277
- meilisearch: { engine: 'meilisearch', category: 'search-index' },
278
- rabbitmq: { engine: 'rabbitmq', category: 'message-queue' },
279
- kafka: { engine: 'kafka', category: 'message-queue' },
280
- nats: { engine: 'nats', category: 'message-queue' },
281
- // Distributed KV / config stores and time-series DBs run as their own service.
282
- etcd: { engine: 'etcd', category: 'key-value-store' },
283
- consul: { engine: 'consul', category: 'key-value-store' },
284
- influxdb: { engine: 'influxdb', category: 'time-series-database' },
285
- });
286
- // URL scheme -> engine, for connection strings in config or source.
287
- // Null-prototype (see LIBRARY_ENGINES): schemes come from arbitrary text.
288
- const URL_SCHEME_ENGINES = Object.assign(Object.create(null), {
289
- 'postgres': { engine: 'postgres', category: 'relational-database' },
290
- 'postgresql': { engine: 'postgres', category: 'relational-database' },
291
- 'mysql': { engine: 'mysql', category: 'relational-database' },
292
- 'mongodb': { engine: 'mongodb', category: 'document-database' },
293
- 'mongodb+srv': { engine: 'mongodb', category: 'document-database' },
294
- 'redis': { engine: 'redis', category: 'key-value-store' },
295
- 'rediss': { engine: 'redis', category: 'key-value-store' },
296
- 'amqp': { engine: 'rabbitmq', category: 'message-queue' },
297
- 'amqps': { engine: 'rabbitmq', category: 'message-queue' },
298
- 'elasticsearch': { engine: 'elasticsearch', category: 'search-index' },
299
- 'clickhouse': { engine: 'clickhouse', category: 'columnar-database' },
300
- 'clickhouses': { engine: 'clickhouse', category: 'columnar-database' },
301
- 'cockroachdb': { engine: 'cockroachdb', category: 'relational-database' },
302
- });
303
- const CONFIDENCE_RANK = {
304
- running: 3, configured: 2, library: 1,
305
- };
306
- function manifestDependencyNames(input) {
307
- const { lowerName, text } = input;
308
- const names = new Set();
309
- if (lowerName === 'package.json') {
310
- try {
311
- const parsed = JSON.parse(text);
312
- for (const field of ['dependencies', 'optionalDependencies', 'peerDependencies']) {
313
- const deps = parsed[field];
314
- if (deps && typeof deps === 'object') {
315
- for (const name of Object.keys(deps))
316
- names.add(name);
317
- }
318
- }
319
- }
320
- catch { /* ignore malformed manifest */ }
321
- return [...names];
322
- }
323
- if (lowerName === 'requirements.txt' || lowerName.startsWith('requirements')) {
324
- for (const line of text.split('\n')) {
325
- const match = line.match(/^\s*([A-Za-z0-9_.-]+)/);
326
- if (match)
327
- names.add(match[1]);
328
- }
329
- return [...names];
330
- }
331
- if (lowerName === 'pyproject.toml' || lowerName === 'go.mod' || lowerName === 'gemfile'
332
- || lowerName === 'composer.json' || lowerName === 'pom.xml'
333
- || lowerName === 'build.gradle' || lowerName === 'build.gradle.kts'
334
- || lowerName === 'mix.exs' || lowerName === 'cargo.toml') {
335
- // Coarse token scan: a store library name appearing anywhere in these
336
- // manifests is a dependency signal. The library map keys are specific
337
- // enough (pg, ioredis, amqplib, cassandra-driver, postgrex, ecto_ch) that
338
- // false matches are rare, and go.mod/pom paths are matched as full module
339
- // coordinates. Elixir deps are `{:postgrex, ...}` atoms, so `{`/`}` split
340
- // too, leaving the bare package name as a token. Only `mix.exs` (direct
341
- // deps) is read, never `mix.lock` — a lockfile is the full transitive tree
342
- // (dev/test deps and their deps), the same reason package-lock.json is
343
- // skipped in favor of package.json. `Cargo.toml` (Rust) is read the same
344
- // way; a store crate's engine-carrying feature (e.g. `features=["postgres"]`)
345
- // matches as its own token, so the adapter-agnostic sqlx/diesel still resolve.
346
- for (const token of text.split(/[\s"',=:(){}[\]]+/)) {
347
- if (token)
348
- names.add(token);
349
- }
350
- return [...names];
351
- }
352
- // .NET project files: PackageReference / PackageVersion Include names. Covers
353
- // .csproj, .fsproj, .vbproj, Directory.Packages.props, Directory.Build.props,
354
- // and packages.config (the whole .NET dependency-manifest surface).
355
- if (lowerName.endsWith('.csproj') || lowerName.endsWith('.fsproj')
356
- || lowerName.endsWith('.vbproj') || lowerName.endsWith('.props')
357
- || lowerName === 'packages.config') {
358
- for (const match of text.matchAll(/<(?:PackageReference|PackageVersion|package)\s+[^>]*\b(?:Include|id)\s*=\s*"([^"]+)"/gi)) {
359
- names.add(match[1]);
360
- }
361
- return [...names];
362
- }
363
- return [];
364
- }
365
- // CI pipeline config paths. A `services:` container declared in a CI workflow
366
- // (GitHub Actions, GitLab, CircleCI, ...) is ephemeral test infrastructure, not
367
- // the application's production topology — a Postgres spun up to run the test
368
- // suite is not a store the app ships with. Scanning these for `image:` invents
369
- // running stores (measured: twenty's ClickHouse came from .github/workflows,
370
- // not its compose). Matched on the normalized repo-relative path, not content.
371
- function isCiConfigPath(path) {
372
- const p = path.toLowerCase().replace(/\\/g, '/');
373
- return p.includes('.github/workflows/')
374
- || p.includes('.gitea/workflows/')
375
- || p.includes('.forgejo/workflows/')
376
- || p.includes('.circleci/')
377
- || p.includes('.buildkite/')
378
- || /(^|\/)\.gitlab-ci\.ya?ml$/.test(p)
379
- || /(^|\/)\.drone\.ya?ml$/.test(p)
380
- || /(^|\/)\.woodpecker\.ya?ml$/.test(p)
381
- || p.includes('/.woodpecker/')
382
- || /(^|\/)bitbucket-pipelines\.ya?ml$/.test(p)
383
- || /(^|\/)azure-pipelines\.ya?ml$/.test(p);
384
- }
385
- function serviceImages(input) {
386
- const { lowerName, text } = input;
387
- const isCompose = /(^|\/)(docker-)?compose\.ya?ml$/.test(lowerName)
388
- || lowerName === 'compose.yml' || lowerName === 'compose.yaml'
389
- || lowerName === 'docker-compose.yml' || lowerName === 'docker-compose.yaml';
390
- const isYaml = lowerName.endsWith('.yml') || lowerName.endsWith('.yaml');
391
- if (!isCompose && !isYaml)
392
- return [];
393
- const images = [];
394
- for (const line of text.split('\n')) {
395
- const match = line.match(/^\s*(?:-\s*)?image\s*:\s*["']?([^"'\s]+)["']?/);
396
- if (match)
397
- images.push(match[1]);
398
- }
399
- return images;
400
- }
401
- function connectionUrlSchemes(input) {
402
- const schemes = [];
403
- for (const match of input.text.matchAll(/\b([a-z][a-z0-9+.-]*)\s*:\/\//gi)) {
404
- schemes.push(match[1].toLowerCase());
405
- }
406
- return schemes;
407
- }
408
- // High-signal config declarations that name a store without an image, a client
409
- // library, or a URL. Two whole ecosystems configure their stores this way —
410
- // Python/Django (`DATABASES … ENGINE`), and PHP (built-in PDO/mysqli named only
411
- // in an .ini/.php config) — plus the conventional `<ENGINE>_HOST` env keys. On
412
- // the survey these were the dominant recall gap (zulip's four stores, matomo's
413
- // database). Kept deliberately specific to avoid matching prose or an example.
414
- const CONFIG_ENGINE_PATTERNS = [
415
- // Django database backends: `django.db.backends.postgresql`
416
- { re: /django\.db\.backends\.postgresql/i, profile: { engine: 'postgres', category: 'relational-database' } },
417
- { re: /django\.db\.backends\.mysql/i, profile: { engine: 'mysql', category: 'relational-database' } },
418
- { re: /django\.db\.backends\.oracle/i, profile: { engine: 'oracle', category: 'relational-database' } },
419
- { re: /django\.db\.backends\.sqlite3/i, profile: { engine: 'sqlite', category: 'embedded-database' } },
420
- // Django cache backends
421
- { re: /django(?:\.core\.cache\.backends\.memcached|_bmemcached|\.core\.cache\.backends\.pymemcache)/i, profile: { engine: 'memcached', category: 'key-value-store' } },
422
- { re: /(?:django_redis|django\.core\.cache\.backends\.redis)/i, profile: { engine: 'redis', category: 'key-value-store' } },
423
- // PHP built-in DB drivers named only in config (matomo, many PHP apps)
424
- { re: /\b(?:adapter\s*=\s*)?PDO\\?_?MYSQL\b/i, profile: { engine: 'mysql', category: 'relational-database' } },
425
- { re: /\b(?:adapter\s*=\s*)?PDO\\?_?PGSQL\b/i, profile: { engine: 'postgres', category: 'relational-database' } },
426
- // Laravel .env: DB_CONNECTION names the built-in PDO driver (firefly, koel)
427
- { re: /\bDB_CONNECTION\s*=\s*["']?(?:mysql|mariadb)\b/i, profile: { engine: 'mysql', category: 'relational-database' } },
428
- { re: /\bDB_CONNECTION\s*=\s*["']?(?:pgsql|postgres)\b/i, profile: { engine: 'postgres', category: 'relational-database' } },
429
- { re: /\bDB_CONNECTION\s*=\s*["']?sqlite\b/i, profile: { engine: 'sqlite', category: 'embedded-database' } },
430
- { re: /\bDB_CONNECTION\s*=\s*["']?sqlsrv\b/i, profile: { engine: 'sqlserver', category: 'relational-database' } },
431
- // Rails config/database.yml: adapter names the driver (chatwoot, many Rails)
432
- { re: /\badapter\s*:\s*postgresql\b/i, profile: { engine: 'postgres', category: 'relational-database' } },
433
- { re: /\badapter\s*:\s*mysql2?\b/i, profile: { engine: 'mysql', category: 'relational-database' } },
434
- { re: /\badapter\s*:\s*sqlite3\b/i, profile: { engine: 'sqlite', category: 'embedded-database' } },
435
- { re: /\badapter\s*:\s*sqlserver\b/i, profile: { engine: 'sqlserver', category: 'relational-database' } },
436
- // Conventional <ENGINE>_HOST / _DATABASE env keys for the SQL engines too
437
- { re: /\bPOSTGRES(?:QL)?_(?:HOST|DB|DATABASE)\b/i, profile: { engine: 'postgres', category: 'relational-database' } },
438
- { re: /\bPG(?:HOST|DATABASE)\b/, profile: { engine: 'postgres', category: 'relational-database' } },
439
- { re: /\bMYSQL_(?:HOST|DATABASE|DB)\b/i, profile: { engine: 'mysql', category: 'relational-database' } },
440
- // Conventional <ENGINE>_HOST / _LOCATION / _SERVERS / _CONTACT_POINTS env keys
441
- { re: /\bREDIS_(?:HOST|HOSTS|SERVER|URL)\b/i, profile: { engine: 'redis', category: 'key-value-store' } },
442
- { re: /\bMEMCACHED_(?:HOST|HOSTS|LOCATION|SERVERS)\b/i, profile: { engine: 'memcached', category: 'key-value-store' } },
443
- { re: /\bRABBITMQ_(?:HOST|HOSTS|URL)\b/i, profile: { engine: 'rabbitmq', category: 'message-queue' } },
444
- { re: /\bKAFKA_(?:BROKERS|BOOTSTRAP|BOOTSTRAP_SERVERS|HOSTS?|SERVERS)\b/i, profile: { engine: 'kafka', category: 'message-queue' } },
445
- { re: /\bCLICKHOUSE_(?:HOST|URL|DSN|ADDR)\b/i, profile: { engine: 'clickhouse', category: 'columnar-database' } },
446
- { re: /\bCASSANDRA_(?:HOST|HOSTS|CONTACT_POINTS|SEEDS)\b/i, profile: { engine: 'cassandra', category: 'wide-column-database' } },
447
- { re: /\bMONGO(?:DB)?_(?:HOST|URL|URI)\b/i, profile: { engine: 'mongodb', category: 'document-database' } },
448
- { re: /\bELASTICSEARCH_(?:HOST|HOSTS|URL)\b/i, profile: { engine: 'elasticsearch', category: 'search-index' } },
449
- { re: /\bETCD_(?:HOST|HOSTS|ENDPOINT|ENDPOINTS)\b/i, profile: { engine: 'etcd', category: 'key-value-store' } },
450
- { re: /\bINFLUX(?:DB)?_(?:HOST|URL|BUCKET|ORG)\b/i, profile: { engine: 'influxdb', category: 'time-series-database' } },
451
- { re: /\bCONSUL_(?:HTTP_ADDR|HOST|URL)\b/i, profile: { engine: 'consul', category: 'key-value-store' } },
452
- ];
453
- function configEngineSignals(input) {
454
- const found = [];
455
- for (const { re, profile } of CONFIG_ENGINE_PATTERNS) {
456
- if (re.test(input.text))
457
- found.push(profile);
458
- }
459
- return found;
460
- }
461
- function imageEngineFor(image) {
462
- // Drop the digest, then test each path segment against the engine map, last
463
- // segment first. `bitnami/postgresql:16` matches on `postgresql`;
464
- // `mcr.microsoft.com/mssql/server` matches on the `mssql` segment (the engine
465
- // is not always the final segment behind a registry path).
466
- const segments = image.split('@')[0].split('/').map((segment) => segment.toLowerCase());
467
- // Strip the tag from the final segment ONLY — a colon in an earlier segment is
468
- // a registry port (`localhost:5000/postgres:16`), not a tag separator.
469
- segments[segments.length - 1] = segments[segments.length - 1].split(':')[0];
470
- for (let index = segments.length - 1; index >= 0; index -= 1) {
471
- const profile = IMAGE_ENGINES[segments[index]];
472
- if (profile)
473
- return profile;
474
- }
475
- return undefined;
476
- }
477
- /**
478
- * Fold a stream of manifest/compose/config files into the distinct data stores
479
- * the application uses. `inputs` is every scanned text file with its lowercased
480
- * basename; the detector reads only the manifests and configs it recognizes.
481
- */
482
- export function detectDataStores(inputs) {
483
- const byEngine = new Map();
484
- const upsert = (profile, confidence, signal) => {
485
- const existing = byEngine.get(profile.engine);
486
- if (existing) {
487
- existing.signals.push(signal);
488
- if (CONFIDENCE_RANK[confidence] > CONFIDENCE_RANK[existing.confidence]) {
489
- existing.confidence = confidence;
490
- }
491
- return;
492
- }
493
- byEngine.set(profile.engine, {
494
- engine: profile.engine,
495
- category: profile.category,
496
- clone_strategy: CLONE_STRATEGY_BY_CATEGORY[profile.category],
497
- confidence,
498
- signals: [signal],
499
- });
500
- };
501
- for (const input of inputs) {
502
- // A CI pipeline file is test infrastructure end to end: its service images,
503
- // its `POSTGRES_HOST`/`REDIS_URL` env, and its connection strings all describe
504
- // the test harness, not the app's production stores. Skip it for every signal
505
- // (it is never a dependency manifest, so nothing of value is lost).
506
- if (isCiConfigPath(input.path))
507
- continue;
508
- for (const image of serviceImages(input)) {
509
- const profile = imageEngineFor(image);
510
- if (profile)
511
- upsert(profile, 'running', { kind: 'service-image', value: image, path: input.path });
512
- }
513
- for (const dependency of manifestDependencyNames(input)) {
514
- const profile = LIBRARY_ENGINES[dependency.toLowerCase()];
515
- if (profile)
516
- upsert(profile, 'library', { kind: 'client-library', value: dependency, path: input.path });
517
- }
518
- for (const scheme of connectionUrlSchemes(input)) {
519
- const profile = URL_SCHEME_ENGINES[scheme];
520
- if (profile)
521
- upsert(profile, 'configured', { kind: 'connection-url', value: `${scheme}://`, path: input.path });
522
- }
523
- for (const profile of configEngineSignals(input)) {
524
- upsert(profile, 'configured', { kind: 'config-declaration', value: profile.engine, path: input.path });
525
- }
526
- }
527
- return [...byEngine.values()]
528
- .map((store) => ({
529
- ...store,
530
- // Cap and stabilize evidence for a deterministic, bounded record.
531
- signals: store.signals
532
- .slice()
533
- .sort((left, right) => left.kind.localeCompare(right.kind) ||
534
- left.path.localeCompare(right.path) ||
535
- left.value.localeCompare(right.value))
536
- .slice(0, 24),
537
- }))
538
- .sort((left, right) => left.engine.localeCompare(right.engine));
539
- }