@haystackeditor/cli 0.25.1 → 0.27.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (109) hide show
  1. package/README.md +39 -463
  2. package/dist/capture/app-config.js +30 -6
  3. package/dist/commands/capture-brief.js +45 -35
  4. package/dist/commands/capture-contract.js +4 -4
  5. package/dist/commands/crawl-report.js +196 -0
  6. package/dist/commands/db-profile-upload.js +3 -3
  7. package/dist/commands/feedback.js +66 -0
  8. package/dist/commands/init-telemetry.js +55 -9
  9. package/dist/commands/init.js +8 -3
  10. package/dist/commands/lockfile-pin.js +307 -0
  11. package/dist/commands/telemetry-token.js +3 -3
  12. package/dist/commands/tokens.js +4 -4
  13. package/dist/commands/verify-explore.js +3 -3
  14. package/dist/commands/verify-history.js +2 -2
  15. package/dist/commands/verify-onboarding.js +13 -16
  16. package/dist/commands/verify-precompute.js +3 -4
  17. package/dist/commands/verify.js +31 -118
  18. package/dist/index.js +55 -968
  19. package/dist/schema.js +4 -10
  20. package/dist/utils/haystack-api.js +8 -36
  21. package/package.json +1 -5
  22. package/schemas/feedback.v1.json +13 -0
  23. package/schemas/pre-verify.v2.json +240 -0
  24. package/schemas/verify-raw.v1.json +1132 -0
  25. package/schemas/verify.v2.json +655 -0
  26. package/dist/assets/hooks/agent-context/detect.ts +0 -316
  27. package/dist/assets/hooks/agent-context/format.ts +0 -100
  28. package/dist/assets/hooks/agent-context/index.ts +0 -41
  29. package/dist/assets/hooks/agent-context/parsers/claude.ts +0 -262
  30. package/dist/assets/hooks/agent-context/parsers/codex.ts +0 -416
  31. package/dist/assets/hooks/agent-context/parsers/gemini.ts +0 -155
  32. package/dist/assets/hooks/agent-context/parsers/opencode.ts +0 -174
  33. package/dist/assets/hooks/agent-context/tsconfig.json +0 -14
  34. package/dist/assets/hooks/agent-context/types.ts +0 -58
  35. package/dist/assets/hooks/llm-rules-template.md +0 -59
  36. package/dist/assets/hooks/package-lock.json +0 -598
  37. package/dist/assets/hooks/package.json +0 -12
  38. package/dist/assets/hooks/scripts/commit-msg.sh +0 -5
  39. package/dist/assets/hooks/scripts/post-commit.sh +0 -5
  40. package/dist/assets/hooks/scripts/pre-commit.sh +0 -175
  41. package/dist/assets/hooks/scripts/pre-push.sh +0 -25
  42. package/dist/assets/hooks/scripts/prepare-commit-msg.sh +0 -5
  43. package/dist/assets/hooks/truncation-checker/ast-analyzer.ts +0 -528
  44. package/dist/assets/hooks/truncation-checker/index.ts +0 -595
  45. package/dist/assets/hooks/truncation-checker/tsconfig.json +0 -13
  46. package/dist/assets/skills/map-cloud-verifier-universe/SKILL.md +0 -2051
  47. package/dist/assets/skills/map-cloud-verifier-universe/agents/openai.yaml +0 -4
  48. package/dist/assets/skills/map-cloud-verifier-universe/references/output-contract.md +0 -3411
  49. package/dist/assets/skills/map-your-system.md +0 -143
  50. package/dist/assets/skills/submit.md +0 -200
  51. package/dist/commands/ask.js +0 -20
  52. package/dist/commands/cloud-verifier-behaviors.js +0 -218
  53. package/dist/commands/cloud-verifier-data-store-census.js +0 -539
  54. package/dist/commands/cloud-verifier-data-store-drift.js +0 -158
  55. package/dist/commands/cloud-verifier-identity-census.js +0 -4060
  56. package/dist/commands/cloud-verifier-materialization.js +0 -704
  57. package/dist/commands/cloud-verifier-pascal-selector-census.js +0 -1382
  58. package/dist/commands/cloud-verifier-python-manifest-selector-census.js +0 -2015
  59. package/dist/commands/cloud-verifier-specialized-operational-census.js +0 -11432
  60. package/dist/commands/cloud-verifier-universe.js +0 -10178
  61. package/dist/commands/config.js +0 -549
  62. package/dist/commands/design-verify.js +0 -311
  63. package/dist/commands/dismiss.js +0 -159
  64. package/dist/commands/hooks.js +0 -226
  65. package/dist/commands/inbox.js +0 -137
  66. package/dist/commands/mcp.js +0 -201
  67. package/dist/commands/policy.js +0 -371
  68. package/dist/commands/pr-status.js +0 -207
  69. package/dist/commands/pr.js +0 -105
  70. package/dist/commands/prepare-universe-review.js +0 -1092
  71. package/dist/commands/production-source-deny-policy.js +0 -100
  72. package/dist/commands/request-review.js +0 -74
  73. package/dist/commands/review.js +0 -191
  74. package/dist/commands/rules.js +0 -98
  75. package/dist/commands/scaffold-provisional-universe.js +0 -806
  76. package/dist/commands/setup.js +0 -1170
  77. package/dist/commands/skills.js +0 -447
  78. package/dist/commands/status.js +0 -35
  79. package/dist/commands/submit.js +0 -745
  80. package/dist/commands/system-map.js +0 -228
  81. package/dist/commands/triage.js +0 -598
  82. package/dist/commands/webhooks.js +0 -241
  83. package/dist/states.js +0 -46
  84. package/dist/tools/detect.js +0 -832
  85. package/dist/triage/astra.js +0 -202
  86. package/dist/triage/prompts.js +0 -188
  87. package/dist/triage/runner.js +0 -200
  88. package/dist/triage/types.js +0 -7
  89. package/dist/types.js +0 -326
  90. package/dist/utils/action-output.js +0 -26
  91. package/dist/utils/analysis-api.js +0 -416
  92. package/dist/utils/config.js +0 -54
  93. package/dist/utils/design-verifier-api.js +0 -294
  94. package/dist/utils/design-verifier-history.js +0 -79
  95. package/dist/utils/design-verifier-result.js +0 -424
  96. package/dist/utils/github-api.js +0 -324
  97. package/dist/utils/pending-state.js +0 -86
  98. package/dist/utils/pr-ref.js +0 -56
  99. package/dist/utils/prompter.js +0 -328
  100. package/schemas/action.v1.json +0 -22
  101. package/schemas/ask.v1.json +0 -40
  102. package/schemas/inbox.v1.json +0 -27
  103. package/schemas/pr-status.v1.json +0 -61
  104. package/schemas/pr.v1.json +0 -97
  105. package/schemas/pr.v3.json +0 -45
  106. package/schemas/setup.v1.json +0 -75
  107. package/schemas/submit.v1.json +0 -90
  108. package/schemas/triage.v1.json +0 -103
  109. package/schemas/triage.v2.json +0 -64
@@ -1,539 +0,0 @@
1
- /**
2
- * Data-store census.
3
- *
4
- * The one job the Cloud Verifier needs from a repository map: what data stores
5
- * does this application use, so we can clone each one into a replica. Not
6
- * processes, not API calls, not code structure — the databases, caches, object
7
- * stores, search indexes, and queues that hold state a test must run against.
8
- *
9
- * Four source signals, strongest first:
10
- * 1. a service image in a compose/k8s manifest (`image: postgres:16`) — a
11
- * store instance the app is wired to run beside, the surest signal;
12
- * 2. a connection URL scheme or conventional env var (`postgres://`,
13
- * `REDIS_URL`) — an instance named in config;
14
- * 3. a config declaration that names a store without an image, library, or URL
15
- * (Django `ENGINE`, PHP PDO, Laravel `DB_CONNECTION`, Rails `adapter:`);
16
- * 4. a client library in a dependency manifest (`pg`, `redis`, `mongoose`) —
17
- * the app talks to that engine.
18
- *
19
- * Signals are folded per engine into one data store, with the evidence, a
20
- * confidence, and how to reproduce it. Everything here is a general convention,
21
- * never a repository-specific constant.
22
- */
23
- // How a replica of this category is made. Durable stores of record are copied
24
- // from a snapshot; caches and search indexes are usually derived and can be
25
- // rebuilt; queues are drained to empty.
26
- const CLONE_STRATEGY_BY_CATEGORY = {
27
- 'relational-database': 'snapshot-restore',
28
- 'document-database': 'snapshot-restore',
29
- 'key-value-store': 'snapshot-restore',
30
- 'wide-column-database': 'snapshot-restore',
31
- 'columnar-database': 'snapshot-restore',
32
- 'graph-database': 'snapshot-restore',
33
- 'vector-database': 'snapshot-restore',
34
- 'object-storage': 'copy-bucket-snapshot',
35
- 'search-index': 'rebuild-from-source',
36
- 'message-queue': 'reset-empty',
37
- 'time-series-database': 'snapshot-restore',
38
- 'embedded-database': 'copy-file',
39
- };
40
- // Client library (as written in a dependency manifest) -> engine. Covers the
41
- // common stores across Node, Python, Go, Ruby, Java, PHP. Keys are matched
42
- // case-insensitively against exact dependency names.
43
- // Null-prototype: keys are matched against arbitrary dependency tokens from a
44
- // manifest, so a token like `constructor`/`toString`/`hasOwnProperty` must NOT
45
- // resolve to an inherited Object.prototype member (that returned a truthy
46
- // non-profile and created a store with an undefined engine — crashed on a repo
47
- // whose manifest contained such a token).
48
- const LIBRARY_ENGINES = Object.create(null);
49
- function registerLibraries(profile, names) {
50
- for (const name of names)
51
- LIBRARY_ENGINES[name.toLowerCase()] = profile;
52
- }
53
- registerLibraries({ engine: 'postgres', category: 'relational-database' }, [
54
- 'pg', 'pg-promise', 'postgres', 'node-postgres', 'psycopg2', 'psycopg2-binary',
55
- 'psycopg', 'asyncpg', 'lib/pq', 'github.com/lib/pq', 'github.com/jackc/pgx',
56
- 'github.com/jackc/pgx/v5', 'pg gem', 'postgresql', 'org.postgresql:postgresql',
57
- ]);
58
- registerLibraries({ engine: 'mysql', category: 'relational-database' }, [
59
- 'mysql', 'mysql2', 'mariadb', 'pymysql', 'mysqlclient', 'aiomysql',
60
- 'github.com/go-sql-driver/mysql', 'mysql-connector-python', 'mysql:mysql-connector-java',
61
- ]);
62
- registerLibraries({ engine: 'sqlserver', category: 'relational-database' }, [
63
- 'mssql', 'tedious', 'pyodbc', 'pymssql', 'github.com/denisenkom/go-mssqldb',
64
- ]);
65
- registerLibraries({ engine: 'oracle', category: 'relational-database' }, [
66
- 'oracledb', 'cx_oracle', 'cx-oracle',
67
- ]);
68
- registerLibraries({ engine: 'mongodb', category: 'document-database' }, [
69
- 'mongodb', 'mongoose', 'pymongo', 'motor', 'mongoengine',
70
- 'go.mongodb.org/mongo-driver', 'mongo', 'mongoid',
71
- ]);
72
- registerLibraries({ engine: 'couchdb', category: 'document-database' }, [
73
- 'nano', 'couchdb', 'pycouchdb', 'cloudant',
74
- ]);
75
- registerLibraries({ engine: 'redis', category: 'key-value-store' }, [
76
- 'redis', 'ioredis', 'redis-py', 'connect-redis', 'go-redis', 'github.com/redis/go-redis',
77
- 'github.com/go-redis/redis', 'hiredis', 'aioredis', 'redis-rb', 'predis/predis', 'jedis',
78
- ]);
79
- registerLibraries({ engine: 'memcached', category: 'key-value-store' }, [
80
- 'memcached', 'memjs', 'pymemcache', 'python-memcached', 'dalli',
81
- ]);
82
- registerLibraries({ engine: 'dynamodb', category: 'key-value-store' }, [
83
- '@aws-sdk/client-dynamodb', 'dynamoose',
84
- ]);
85
- registerLibraries({ engine: 'cassandra', category: 'wide-column-database' }, [
86
- 'cassandra-driver', 'cassandra', 'scylla-driver', 'datastax',
87
- ]);
88
- registerLibraries({ engine: 'clickhouse', category: 'columnar-database' }, [
89
- '@clickhouse/client', '@clickhouse/client-web', 'clickhouse-driver',
90
- 'clickhouse-connect', 'clickhouse', 'ecto_ch', 'clickhouse_ecto',
91
- 'github.com/clickhouse/clickhouse-go', 'clickhouse-sqlalchemy',
92
- ]);
93
- registerLibraries({ engine: 'cockroachdb', category: 'relational-database' }, [
94
- 'cockroachdb', 'sqlalchemy-cockroachdb',
95
- ]);
96
- registerLibraries({ engine: 'neo4j', category: 'graph-database' }, [
97
- 'neo4j-driver', 'neo4j', 'py2neo', 'neo4j-python-driver',
98
- ]);
99
- registerLibraries({ engine: 'weaviate', category: 'vector-database' }, [
100
- 'weaviate-client', 'weaviate-ts-client', '@weaviate/client', 'weaviate',
101
- ]);
102
- registerLibraries({ engine: 'qdrant', category: 'vector-database' }, [
103
- 'qdrant-client', '@qdrant/js-client-rest', 'qdrant',
104
- ]);
105
- registerLibraries({ engine: 'milvus', category: 'vector-database' }, [
106
- 'pymilvus', '@zilliz/milvus2-sdk-node', 'milvus',
107
- ]);
108
- registerLibraries({ engine: 'pinecone', category: 'vector-database' }, [
109
- 'pinecone-client', '@pinecone-database/pinecone', 'pinecone',
110
- ]);
111
- registerLibraries({ engine: 'chroma', category: 'vector-database' }, [
112
- 'chromadb', 'chromadb-client',
113
- ]);
114
- // Object-storage clients must be the specific S3 client, not a monolithic
115
- // cloud SDK: bare `aws-sdk`/`boto3`/`aws-sdk-go` are used for SES, SNS, SQS,
116
- // and dozens of non-storage services, so mapping them to S3 fabricates stores
117
- // (measured on medusa, saleor, n8n). Only the dedicated S3 client counts.
118
- registerLibraries({ engine: 's3', category: 'object-storage' }, [
119
- '@aws-sdk/client-s3', 'minio', 'aws-sdk-s3',
120
- ]);
121
- registerLibraries({ engine: 'gcs', category: 'object-storage' }, [
122
- '@google-cloud/storage', 'google-cloud-storage', 'gcloud-storage',
123
- ]);
124
- registerLibraries({ engine: 'azure-blob', category: 'object-storage' }, [
125
- '@azure/storage-blob', 'azure-storage-blob', 'azure-storage',
126
- ]);
127
- registerLibraries({ engine: 'elasticsearch', category: 'search-index' }, [
128
- '@elastic/elasticsearch', 'elasticsearch', 'elasticsearch-py', 'elasticsearch-dsl',
129
- '@opensearch-project/opensearch', 'opensearch-py', 'github.com/elastic/go-elasticsearch',
130
- ]);
131
- registerLibraries({ engine: 'solr', category: 'search-index' }, [
132
- 'solr-client', 'pysolr', 'solr',
133
- ]);
134
- registerLibraries({ engine: 'meilisearch', category: 'search-index' }, [
135
- 'meilisearch', 'meilisearch-python',
136
- ]);
137
- registerLibraries({ engine: 'rabbitmq', category: 'message-queue' }, [
138
- 'amqplib', 'amqp', 'pika', 'bunny', 'github.com/streadway/amqp', 'github.com/rabbitmq/amqp091-go',
139
- ]);
140
- registerLibraries({ engine: 'kafka', category: 'message-queue' }, [
141
- 'kafkajs', 'node-rdkafka', 'kafka-python', 'confluent-kafka', 'github.com/segmentio/kafka-go',
142
- 'sarama', 'github.com/ibm/sarama',
143
- ]);
144
- registerLibraries({ engine: 'sqs', category: 'message-queue' }, [
145
- '@aws-sdk/client-sqs', 'boto3-sqs',
146
- ]);
147
- registerLibraries({ engine: 'nats', category: 'message-queue' }, [
148
- 'nats', 'nats.js', 'nats-py', 'github.com/nats-io/nats.go',
149
- ]);
150
- // Distributed key-value / config stores that hold an app's own coordination
151
- // state (apisix keeps its routes in etcd; vault/apisix can use consul). Keys are
152
- // specific client coordinates — bare `consul`/`etcd` are avoided as too generic.
153
- registerLibraries({ engine: 'etcd', category: 'key-value-store' }, [
154
- 'etcd3', 'python-etcd3', 'go.etcd.io/etcd', 'go.etcd.io/etcd/client/v3',
155
- 'github.com/coreos/etcd', 'coreos-etcd',
156
- ]);
157
- registerLibraries({ engine: 'consul', category: 'key-value-store' }, [
158
- 'python-consul', 'python-consul2', 'github.com/hashicorp/consul/api', 'node-consul',
159
- ]);
160
- // Time-series databases (flagsmith stores analytics in InfluxDB).
161
- registerLibraries({ engine: 'influxdb', category: 'time-series-database' }, [
162
- 'influxdb-client', 'influxdb', '@influxdata/influxdb-client', 'influxdb-python',
163
- // Go clients are imported at their versioned module path.
164
- 'github.com/influxdata/influxdb-client-go', 'github.com/influxdata/influxdb-client-go/v2',
165
- 'github.com/influxdata/influxdb1-client', 'github.com/influxdata/influxdb1-client/v2',
166
- ]);
167
- // Redis-backed job/queue libraries are NOT a separate store — their queue
168
- // lives inside Redis. Mapping them to `redis` folds them into the broker
169
- // (measured: they created a phantom `redis-queue` alongside redis in 8 repos).
170
- // `celery` is omitted because its broker is configurable (Redis or RabbitMQ),
171
- // and the real broker is detected from its own client or a connection URL.
172
- registerLibraries({ engine: 'redis', category: 'key-value-store' }, [
173
- 'bull', 'bullmq', 'bee-queue', 'sidekiq', 'resque', 'rq',
174
- ]);
175
- registerLibraries({ engine: 'sqlite', category: 'embedded-database' }, [
176
- 'sqlite3', 'better-sqlite3', 'sqlite', 'aiosqlite',
177
- 'github.com/mattn/go-sqlite3', 'modernc.org/sqlite', 'github.com/glebarez/sqlite',
178
- 'gorm.io/driver/sqlite',
179
- // .NET
180
- 'microsoft.data.sqlite', 'microsoft.entityframeworkcore.sqlite', 'system.data.sqlite',
181
- ]);
182
- // .NET data-access packages (from .csproj / Directory.Packages.props)
183
- registerLibraries({ engine: 'postgres', category: 'relational-database' }, [
184
- 'npgsql', 'npgsql.entityframeworkcore.postgresql',
185
- ]);
186
- registerLibraries({ engine: 'mysql', category: 'relational-database' }, [
187
- 'mysqlconnector', 'mysql.data', 'pomelo.entityframeworkcore.mysql',
188
- ]);
189
- registerLibraries({ engine: 'sqlserver', category: 'relational-database' }, [
190
- 'microsoft.data.sqlclient', 'system.data.sqlclient',
191
- 'microsoft.entityframeworkcore.sqlserver',
192
- ]);
193
- registerLibraries({ engine: 'oracle', category: 'relational-database' }, [
194
- 'oracle.manageddataaccess', 'oracle.manageddataaccess.core',
195
- ]);
196
- registerLibraries({ engine: 'mongodb', category: 'document-database' }, [
197
- 'mongodb.driver',
198
- ]);
199
- registerLibraries({ engine: 'redis', category: 'key-value-store' }, [
200
- 'stackexchange.redis',
201
- ]);
202
- // ORM/driver packages that pin a specific SQL engine.
203
- registerLibraries({ engine: 'postgres', category: 'relational-database' }, [
204
- 'gorm.io/driver/postgres', 'github.com/uptrace/bun/dialect/pgdialect',
205
- ]);
206
- registerLibraries({ engine: 'mysql', category: 'relational-database' }, [
207
- 'gorm.io/driver/mysql',
208
- ]);
209
- // Elixir Hex drivers (from mix.exs / mix.lock). Each is engine-specific; the
210
- // adapter-agnostic `ecto`/`ecto_sql` are deliberately omitted (they name no
211
- // engine on their own). `ecto_ch`/`clickhouse_ecto` are already in the
212
- // ClickHouse set above.
213
- registerLibraries({ engine: 'postgres', category: 'relational-database' }, ['postgrex']);
214
- registerLibraries({ engine: 'mysql', category: 'relational-database' }, ['myxql']);
215
- registerLibraries({ engine: 'redis', category: 'key-value-store' }, ['redix']);
216
- registerLibraries({ engine: 'mongodb', category: 'document-database' }, ['mongodb_driver']);
217
- registerLibraries({ engine: 'cassandra', category: 'wide-column-database' }, ['xandra']);
218
- // Rust crates (from Cargo.toml). Only engine-specific drivers are named; the
219
- // adapter-agnostic `sqlx`/`diesel`/`sea-orm` are omitted (they name no engine on
220
- // their own — their enabled feature, e.g. "postgres", carries that signal and is
221
- // matched as its own token). The generic `postgres`/`mysql`/`redis`/`mongodb`/
222
- // `clickhouse`/`elasticsearch` crate names already match existing library keys.
223
- registerLibraries({ engine: 'postgres', category: 'relational-database' }, [
224
- 'tokio-postgres', 'deadpool-postgres', 'bb8-postgres', 'sqlx-postgres',
225
- ]);
226
- registerLibraries({ engine: 'mysql', category: 'relational-database' }, ['mysql_async', 'sqlx-mysql']);
227
- registerLibraries({ engine: 'sqlite', category: 'embedded-database' }, ['rusqlite']);
228
- // `fred` is deliberately omitted — it is a common word (author names, comments)
229
- // and the coarse token scan is global across all manifests, so it would mint
230
- // phantom Redis stores. The `-redis`-suffixed crates are unambiguous.
231
- registerLibraries({ engine: 'redis', category: 'key-value-store' }, ['deadpool-redis', 'bb8-redis']);
232
- registerLibraries({ engine: 'kafka', category: 'message-queue' }, ['rdkafka', 'rskafka']);
233
- registerLibraries({ engine: 'rabbitmq', category: 'message-queue' }, ['lapin']);
234
- registerLibraries({ engine: 'cassandra', category: 'wide-column-database' }, ['cdrs-tokio', 'scylla']);
235
- // Container image base name -> engine. Matched against the image before its
236
- // tag and registry path (`postgres:16`, `bitnami/redis`).
237
- // Null-prototype (see LIBRARY_ENGINES): image path segments are arbitrary text.
238
- const IMAGE_ENGINES = Object.assign(Object.create(null), {
239
- postgres: { engine: 'postgres', category: 'relational-database' },
240
- postgresql: { engine: 'postgres', category: 'relational-database' },
241
- postgis: { engine: 'postgres', category: 'relational-database' },
242
- // Postgres distributions/extensions: the store to clone is still Postgres.
243
- pgvector: { engine: 'postgres', category: 'relational-database' },
244
- 'pgvecto-rs': { engine: 'postgres', category: 'relational-database' },
245
- timescaledb: { engine: 'postgres', category: 'relational-database' },
246
- supabase: { engine: 'postgres', category: 'relational-database' },
247
- // Vector databases
248
- weaviate: { engine: 'weaviate', category: 'vector-database' },
249
- qdrant: { engine: 'qdrant', category: 'vector-database' },
250
- milvus: { engine: 'milvus', category: 'vector-database' },
251
- chroma: { engine: 'chroma', category: 'vector-database' },
252
- chromadb: { engine: 'chroma', category: 'vector-database' },
253
- mysql: { engine: 'mysql', category: 'relational-database' },
254
- mariadb: { engine: 'mysql', category: 'relational-database' },
255
- mssql: { engine: 'sqlserver', category: 'relational-database' },
256
- 'mssql-server': { engine: 'sqlserver', category: 'relational-database' },
257
- 'azure-sql-edge': { engine: 'sqlserver', category: 'relational-database' },
258
- mongo: { engine: 'mongodb', category: 'document-database' },
259
- mongodb: { engine: 'mongodb', category: 'document-database' },
260
- couchdb: { engine: 'couchdb', category: 'document-database' },
261
- redis: { engine: 'redis', category: 'key-value-store' },
262
- valkey: { engine: 'redis', category: 'key-value-store' },
263
- clickhouse: { engine: 'clickhouse', category: 'columnar-database' },
264
- 'clickhouse-server': { engine: 'clickhouse', category: 'columnar-database' },
265
- cockroachdb: { engine: 'cockroachdb', category: 'relational-database' },
266
- cockroach: { engine: 'cockroachdb', category: 'relational-database' },
267
- azurite: { engine: 'azure-blob', category: 'object-storage' },
268
- 'servicebus-emulator': { engine: 'azure-servicebus', category: 'message-queue' },
269
- memcached: { engine: 'memcached', category: 'key-value-store' },
270
- cassandra: { engine: 'cassandra', category: 'wide-column-database' },
271
- scylladb: { engine: 'cassandra', category: 'wide-column-database' },
272
- neo4j: { engine: 'neo4j', category: 'graph-database' },
273
- minio: { engine: 's3', category: 'object-storage' },
274
- elasticsearch: { engine: 'elasticsearch', category: 'search-index' },
275
- opensearchproject: { engine: 'elasticsearch', category: 'search-index' },
276
- solr: { engine: 'solr', category: 'search-index' },
277
- meilisearch: { engine: 'meilisearch', category: 'search-index' },
278
- rabbitmq: { engine: 'rabbitmq', category: 'message-queue' },
279
- kafka: { engine: 'kafka', category: 'message-queue' },
280
- nats: { engine: 'nats', category: 'message-queue' },
281
- // Distributed KV / config stores and time-series DBs run as their own service.
282
- etcd: { engine: 'etcd', category: 'key-value-store' },
283
- consul: { engine: 'consul', category: 'key-value-store' },
284
- influxdb: { engine: 'influxdb', category: 'time-series-database' },
285
- });
286
- // URL scheme -> engine, for connection strings in config or source.
287
- // Null-prototype (see LIBRARY_ENGINES): schemes come from arbitrary text.
288
- const URL_SCHEME_ENGINES = Object.assign(Object.create(null), {
289
- 'postgres': { engine: 'postgres', category: 'relational-database' },
290
- 'postgresql': { engine: 'postgres', category: 'relational-database' },
291
- 'mysql': { engine: 'mysql', category: 'relational-database' },
292
- 'mongodb': { engine: 'mongodb', category: 'document-database' },
293
- 'mongodb+srv': { engine: 'mongodb', category: 'document-database' },
294
- 'redis': { engine: 'redis', category: 'key-value-store' },
295
- 'rediss': { engine: 'redis', category: 'key-value-store' },
296
- 'amqp': { engine: 'rabbitmq', category: 'message-queue' },
297
- 'amqps': { engine: 'rabbitmq', category: 'message-queue' },
298
- 'elasticsearch': { engine: 'elasticsearch', category: 'search-index' },
299
- 'clickhouse': { engine: 'clickhouse', category: 'columnar-database' },
300
- 'clickhouses': { engine: 'clickhouse', category: 'columnar-database' },
301
- 'cockroachdb': { engine: 'cockroachdb', category: 'relational-database' },
302
- });
303
- const CONFIDENCE_RANK = {
304
- running: 3, configured: 2, library: 1,
305
- };
306
- function manifestDependencyNames(input) {
307
- const { lowerName, text } = input;
308
- const names = new Set();
309
- if (lowerName === 'package.json') {
310
- try {
311
- const parsed = JSON.parse(text);
312
- for (const field of ['dependencies', 'optionalDependencies', 'peerDependencies']) {
313
- const deps = parsed[field];
314
- if (deps && typeof deps === 'object') {
315
- for (const name of Object.keys(deps))
316
- names.add(name);
317
- }
318
- }
319
- }
320
- catch { /* ignore malformed manifest */ }
321
- return [...names];
322
- }
323
- if (lowerName === 'requirements.txt' || lowerName.startsWith('requirements')) {
324
- for (const line of text.split('\n')) {
325
- const match = line.match(/^\s*([A-Za-z0-9_.-]+)/);
326
- if (match)
327
- names.add(match[1]);
328
- }
329
- return [...names];
330
- }
331
- if (lowerName === 'pyproject.toml' || lowerName === 'go.mod' || lowerName === 'gemfile'
332
- || lowerName === 'composer.json' || lowerName === 'pom.xml'
333
- || lowerName === 'build.gradle' || lowerName === 'build.gradle.kts'
334
- || lowerName === 'mix.exs' || lowerName === 'cargo.toml') {
335
- // Coarse token scan: a store library name appearing anywhere in these
336
- // manifests is a dependency signal. The library map keys are specific
337
- // enough (pg, ioredis, amqplib, cassandra-driver, postgrex, ecto_ch) that
338
- // false matches are rare, and go.mod/pom paths are matched as full module
339
- // coordinates. Elixir deps are `{:postgrex, ...}` atoms, so `{`/`}` split
340
- // too, leaving the bare package name as a token. Only `mix.exs` (direct
341
- // deps) is read, never `mix.lock` — a lockfile is the full transitive tree
342
- // (dev/test deps and their deps), the same reason package-lock.json is
343
- // skipped in favor of package.json. `Cargo.toml` (Rust) is read the same
344
- // way; a store crate's engine-carrying feature (e.g. `features=["postgres"]`)
345
- // matches as its own token, so the adapter-agnostic sqlx/diesel still resolve.
346
- for (const token of text.split(/[\s"',=:(){}[\]]+/)) {
347
- if (token)
348
- names.add(token);
349
- }
350
- return [...names];
351
- }
352
- // .NET project files: PackageReference / PackageVersion Include names. Covers
353
- // .csproj, .fsproj, .vbproj, Directory.Packages.props, Directory.Build.props,
354
- // and packages.config (the whole .NET dependency-manifest surface).
355
- if (lowerName.endsWith('.csproj') || lowerName.endsWith('.fsproj')
356
- || lowerName.endsWith('.vbproj') || lowerName.endsWith('.props')
357
- || lowerName === 'packages.config') {
358
- for (const match of text.matchAll(/<(?:PackageReference|PackageVersion|package)\s+[^>]*\b(?:Include|id)\s*=\s*"([^"]+)"/gi)) {
359
- names.add(match[1]);
360
- }
361
- return [...names];
362
- }
363
- return [];
364
- }
365
- // CI pipeline config paths. A `services:` container declared in a CI workflow
366
- // (GitHub Actions, GitLab, CircleCI, ...) is ephemeral test infrastructure, not
367
- // the application's production topology — a Postgres spun up to run the test
368
- // suite is not a store the app ships with. Scanning these for `image:` invents
369
- // running stores (measured: twenty's ClickHouse came from .github/workflows,
370
- // not its compose). Matched on the normalized repo-relative path, not content.
371
- function isCiConfigPath(path) {
372
- const p = path.toLowerCase().replace(/\\/g, '/');
373
- return p.includes('.github/workflows/')
374
- || p.includes('.gitea/workflows/')
375
- || p.includes('.forgejo/workflows/')
376
- || p.includes('.circleci/')
377
- || p.includes('.buildkite/')
378
- || /(^|\/)\.gitlab-ci\.ya?ml$/.test(p)
379
- || /(^|\/)\.drone\.ya?ml$/.test(p)
380
- || /(^|\/)\.woodpecker\.ya?ml$/.test(p)
381
- || p.includes('/.woodpecker/')
382
- || /(^|\/)bitbucket-pipelines\.ya?ml$/.test(p)
383
- || /(^|\/)azure-pipelines\.ya?ml$/.test(p);
384
- }
385
- function serviceImages(input) {
386
- const { lowerName, text } = input;
387
- const isCompose = /(^|\/)(docker-)?compose\.ya?ml$/.test(lowerName)
388
- || lowerName === 'compose.yml' || lowerName === 'compose.yaml'
389
- || lowerName === 'docker-compose.yml' || lowerName === 'docker-compose.yaml';
390
- const isYaml = lowerName.endsWith('.yml') || lowerName.endsWith('.yaml');
391
- if (!isCompose && !isYaml)
392
- return [];
393
- const images = [];
394
- for (const line of text.split('\n')) {
395
- const match = line.match(/^\s*(?:-\s*)?image\s*:\s*["']?([^"'\s]+)["']?/);
396
- if (match)
397
- images.push(match[1]);
398
- }
399
- return images;
400
- }
401
- function connectionUrlSchemes(input) {
402
- const schemes = [];
403
- for (const match of input.text.matchAll(/\b([a-z][a-z0-9+.-]*)\s*:\/\//gi)) {
404
- schemes.push(match[1].toLowerCase());
405
- }
406
- return schemes;
407
- }
408
- // High-signal config declarations that name a store without an image, a client
409
- // library, or a URL. Two whole ecosystems configure their stores this way —
410
- // Python/Django (`DATABASES … ENGINE`), and PHP (built-in PDO/mysqli named only
411
- // in an .ini/.php config) — plus the conventional `<ENGINE>_HOST` env keys. On
412
- // the survey these were the dominant recall gap (zulip's four stores, matomo's
413
- // database). Kept deliberately specific to avoid matching prose or an example.
414
- const CONFIG_ENGINE_PATTERNS = [
415
- // Django database backends: `django.db.backends.postgresql`
416
- { re: /django\.db\.backends\.postgresql/i, profile: { engine: 'postgres', category: 'relational-database' } },
417
- { re: /django\.db\.backends\.mysql/i, profile: { engine: 'mysql', category: 'relational-database' } },
418
- { re: /django\.db\.backends\.oracle/i, profile: { engine: 'oracle', category: 'relational-database' } },
419
- { re: /django\.db\.backends\.sqlite3/i, profile: { engine: 'sqlite', category: 'embedded-database' } },
420
- // Django cache backends
421
- { re: /django(?:\.core\.cache\.backends\.memcached|_bmemcached|\.core\.cache\.backends\.pymemcache)/i, profile: { engine: 'memcached', category: 'key-value-store' } },
422
- { re: /(?:django_redis|django\.core\.cache\.backends\.redis)/i, profile: { engine: 'redis', category: 'key-value-store' } },
423
- // PHP built-in DB drivers named only in config (matomo, many PHP apps)
424
- { re: /\b(?:adapter\s*=\s*)?PDO\\?_?MYSQL\b/i, profile: { engine: 'mysql', category: 'relational-database' } },
425
- { re: /\b(?:adapter\s*=\s*)?PDO\\?_?PGSQL\b/i, profile: { engine: 'postgres', category: 'relational-database' } },
426
- // Laravel .env: DB_CONNECTION names the built-in PDO driver (firefly, koel)
427
- { re: /\bDB_CONNECTION\s*=\s*["']?(?:mysql|mariadb)\b/i, profile: { engine: 'mysql', category: 'relational-database' } },
428
- { re: /\bDB_CONNECTION\s*=\s*["']?(?:pgsql|postgres)\b/i, profile: { engine: 'postgres', category: 'relational-database' } },
429
- { re: /\bDB_CONNECTION\s*=\s*["']?sqlite\b/i, profile: { engine: 'sqlite', category: 'embedded-database' } },
430
- { re: /\bDB_CONNECTION\s*=\s*["']?sqlsrv\b/i, profile: { engine: 'sqlserver', category: 'relational-database' } },
431
- // Rails config/database.yml: adapter names the driver (chatwoot, many Rails)
432
- { re: /\badapter\s*:\s*postgresql\b/i, profile: { engine: 'postgres', category: 'relational-database' } },
433
- { re: /\badapter\s*:\s*mysql2?\b/i, profile: { engine: 'mysql', category: 'relational-database' } },
434
- { re: /\badapter\s*:\s*sqlite3\b/i, profile: { engine: 'sqlite', category: 'embedded-database' } },
435
- { re: /\badapter\s*:\s*sqlserver\b/i, profile: { engine: 'sqlserver', category: 'relational-database' } },
436
- // Conventional <ENGINE>_HOST / _DATABASE env keys for the SQL engines too
437
- { re: /\bPOSTGRES(?:QL)?_(?:HOST|DB|DATABASE)\b/i, profile: { engine: 'postgres', category: 'relational-database' } },
438
- { re: /\bPG(?:HOST|DATABASE)\b/, profile: { engine: 'postgres', category: 'relational-database' } },
439
- { re: /\bMYSQL_(?:HOST|DATABASE|DB)\b/i, profile: { engine: 'mysql', category: 'relational-database' } },
440
- // Conventional <ENGINE>_HOST / _LOCATION / _SERVERS / _CONTACT_POINTS env keys
441
- { re: /\bREDIS_(?:HOST|HOSTS|SERVER|URL)\b/i, profile: { engine: 'redis', category: 'key-value-store' } },
442
- { re: /\bMEMCACHED_(?:HOST|HOSTS|LOCATION|SERVERS)\b/i, profile: { engine: 'memcached', category: 'key-value-store' } },
443
- { re: /\bRABBITMQ_(?:HOST|HOSTS|URL)\b/i, profile: { engine: 'rabbitmq', category: 'message-queue' } },
444
- { re: /\bKAFKA_(?:BROKERS|BOOTSTRAP|BOOTSTRAP_SERVERS|HOSTS?|SERVERS)\b/i, profile: { engine: 'kafka', category: 'message-queue' } },
445
- { re: /\bCLICKHOUSE_(?:HOST|URL|DSN|ADDR)\b/i, profile: { engine: 'clickhouse', category: 'columnar-database' } },
446
- { re: /\bCASSANDRA_(?:HOST|HOSTS|CONTACT_POINTS|SEEDS)\b/i, profile: { engine: 'cassandra', category: 'wide-column-database' } },
447
- { re: /\bMONGO(?:DB)?_(?:HOST|URL|URI)\b/i, profile: { engine: 'mongodb', category: 'document-database' } },
448
- { re: /\bELASTICSEARCH_(?:HOST|HOSTS|URL)\b/i, profile: { engine: 'elasticsearch', category: 'search-index' } },
449
- { re: /\bETCD_(?:HOST|HOSTS|ENDPOINT|ENDPOINTS)\b/i, profile: { engine: 'etcd', category: 'key-value-store' } },
450
- { re: /\bINFLUX(?:DB)?_(?:HOST|URL|BUCKET|ORG)\b/i, profile: { engine: 'influxdb', category: 'time-series-database' } },
451
- { re: /\bCONSUL_(?:HTTP_ADDR|HOST|URL)\b/i, profile: { engine: 'consul', category: 'key-value-store' } },
452
- ];
453
- function configEngineSignals(input) {
454
- const found = [];
455
- for (const { re, profile } of CONFIG_ENGINE_PATTERNS) {
456
- if (re.test(input.text))
457
- found.push(profile);
458
- }
459
- return found;
460
- }
461
- function imageEngineFor(image) {
462
- // Drop the digest, then test each path segment against the engine map, last
463
- // segment first. `bitnami/postgresql:16` matches on `postgresql`;
464
- // `mcr.microsoft.com/mssql/server` matches on the `mssql` segment (the engine
465
- // is not always the final segment behind a registry path).
466
- const segments = image.split('@')[0].split('/').map((segment) => segment.toLowerCase());
467
- // Strip the tag from the final segment ONLY — a colon in an earlier segment is
468
- // a registry port (`localhost:5000/postgres:16`), not a tag separator.
469
- segments[segments.length - 1] = segments[segments.length - 1].split(':')[0];
470
- for (let index = segments.length - 1; index >= 0; index -= 1) {
471
- const profile = IMAGE_ENGINES[segments[index]];
472
- if (profile)
473
- return profile;
474
- }
475
- return undefined;
476
- }
477
- /**
478
- * Fold a stream of manifest/compose/config files into the distinct data stores
479
- * the application uses. `inputs` is every scanned text file with its lowercased
480
- * basename; the detector reads only the manifests and configs it recognizes.
481
- */
482
- export function detectDataStores(inputs) {
483
- const byEngine = new Map();
484
- const upsert = (profile, confidence, signal) => {
485
- const existing = byEngine.get(profile.engine);
486
- if (existing) {
487
- existing.signals.push(signal);
488
- if (CONFIDENCE_RANK[confidence] > CONFIDENCE_RANK[existing.confidence]) {
489
- existing.confidence = confidence;
490
- }
491
- return;
492
- }
493
- byEngine.set(profile.engine, {
494
- engine: profile.engine,
495
- category: profile.category,
496
- clone_strategy: CLONE_STRATEGY_BY_CATEGORY[profile.category],
497
- confidence,
498
- signals: [signal],
499
- });
500
- };
501
- for (const input of inputs) {
502
- // A CI pipeline file is test infrastructure end to end: its service images,
503
- // its `POSTGRES_HOST`/`REDIS_URL` env, and its connection strings all describe
504
- // the test harness, not the app's production stores. Skip it for every signal
505
- // (it is never a dependency manifest, so nothing of value is lost).
506
- if (isCiConfigPath(input.path))
507
- continue;
508
- for (const image of serviceImages(input)) {
509
- const profile = imageEngineFor(image);
510
- if (profile)
511
- upsert(profile, 'running', { kind: 'service-image', value: image, path: input.path });
512
- }
513
- for (const dependency of manifestDependencyNames(input)) {
514
- const profile = LIBRARY_ENGINES[dependency.toLowerCase()];
515
- if (profile)
516
- upsert(profile, 'library', { kind: 'client-library', value: dependency, path: input.path });
517
- }
518
- for (const scheme of connectionUrlSchemes(input)) {
519
- const profile = URL_SCHEME_ENGINES[scheme];
520
- if (profile)
521
- upsert(profile, 'configured', { kind: 'connection-url', value: `${scheme}://`, path: input.path });
522
- }
523
- for (const profile of configEngineSignals(input)) {
524
- upsert(profile, 'configured', { kind: 'config-declaration', value: profile.engine, path: input.path });
525
- }
526
- }
527
- return [...byEngine.values()]
528
- .map((store) => ({
529
- ...store,
530
- // Cap and stabilize evidence for a deterministic, bounded record.
531
- signals: store.signals
532
- .slice()
533
- .sort((left, right) => left.kind.localeCompare(right.kind) ||
534
- left.path.localeCompare(right.path) ||
535
- left.value.localeCompare(right.value))
536
- .slice(0, 24),
537
- }))
538
- .sort((left, right) => left.engine.localeCompare(right.engine));
539
- }