@haystackeditor/cli 0.25.1 → 0.26.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (97) hide show
  1. package/README.md +39 -455
  2. package/dist/capture/app-config.js +25 -1
  3. package/dist/commands/capture-brief.js +41 -32
  4. package/dist/commands/feedback.js +66 -0
  5. package/dist/commands/init-telemetry.js +53 -7
  6. package/dist/commands/init.js +7 -2
  7. package/dist/commands/lockfile-pin.js +307 -0
  8. package/dist/commands/verify.js +68 -13
  9. package/dist/index.js +27 -923
  10. package/dist/schema.js +4 -10
  11. package/dist/utils/haystack-api.js +0 -36
  12. package/package.json +1 -5
  13. package/schemas/feedback.v1.json +13 -0
  14. package/schemas/pre-verify.v2.json +240 -0
  15. package/schemas/verify-raw.v1.json +1132 -0
  16. package/schemas/verify.v2.json +655 -0
  17. package/dist/assets/hooks/agent-context/detect.ts +0 -316
  18. package/dist/assets/hooks/agent-context/format.ts +0 -100
  19. package/dist/assets/hooks/agent-context/index.ts +0 -41
  20. package/dist/assets/hooks/agent-context/parsers/claude.ts +0 -262
  21. package/dist/assets/hooks/agent-context/parsers/codex.ts +0 -416
  22. package/dist/assets/hooks/agent-context/parsers/gemini.ts +0 -155
  23. package/dist/assets/hooks/agent-context/parsers/opencode.ts +0 -174
  24. package/dist/assets/hooks/agent-context/tsconfig.json +0 -14
  25. package/dist/assets/hooks/agent-context/types.ts +0 -58
  26. package/dist/assets/hooks/llm-rules-template.md +0 -59
  27. package/dist/assets/hooks/package-lock.json +0 -598
  28. package/dist/assets/hooks/package.json +0 -12
  29. package/dist/assets/hooks/scripts/commit-msg.sh +0 -5
  30. package/dist/assets/hooks/scripts/post-commit.sh +0 -5
  31. package/dist/assets/hooks/scripts/pre-commit.sh +0 -175
  32. package/dist/assets/hooks/scripts/pre-push.sh +0 -25
  33. package/dist/assets/hooks/scripts/prepare-commit-msg.sh +0 -5
  34. package/dist/assets/hooks/truncation-checker/ast-analyzer.ts +0 -528
  35. package/dist/assets/hooks/truncation-checker/index.ts +0 -595
  36. package/dist/assets/hooks/truncation-checker/tsconfig.json +0 -13
  37. package/dist/assets/skills/map-cloud-verifier-universe/SKILL.md +0 -2051
  38. package/dist/assets/skills/map-cloud-verifier-universe/agents/openai.yaml +0 -4
  39. package/dist/assets/skills/map-cloud-verifier-universe/references/output-contract.md +0 -3411
  40. package/dist/assets/skills/map-your-system.md +0 -143
  41. package/dist/assets/skills/submit.md +0 -200
  42. package/dist/commands/ask.js +0 -20
  43. package/dist/commands/cloud-verifier-behaviors.js +0 -218
  44. package/dist/commands/cloud-verifier-data-store-census.js +0 -539
  45. package/dist/commands/cloud-verifier-data-store-drift.js +0 -158
  46. package/dist/commands/cloud-verifier-identity-census.js +0 -4060
  47. package/dist/commands/cloud-verifier-materialization.js +0 -704
  48. package/dist/commands/cloud-verifier-pascal-selector-census.js +0 -1382
  49. package/dist/commands/cloud-verifier-python-manifest-selector-census.js +0 -2015
  50. package/dist/commands/cloud-verifier-specialized-operational-census.js +0 -11432
  51. package/dist/commands/cloud-verifier-universe.js +0 -10178
  52. package/dist/commands/config.js +0 -549
  53. package/dist/commands/design-verify.js +0 -311
  54. package/dist/commands/dismiss.js +0 -159
  55. package/dist/commands/hooks.js +0 -226
  56. package/dist/commands/inbox.js +0 -137
  57. package/dist/commands/mcp.js +0 -201
  58. package/dist/commands/policy.js +0 -371
  59. package/dist/commands/pr-status.js +0 -207
  60. package/dist/commands/pr.js +0 -105
  61. package/dist/commands/prepare-universe-review.js +0 -1092
  62. package/dist/commands/production-source-deny-policy.js +0 -100
  63. package/dist/commands/request-review.js +0 -74
  64. package/dist/commands/review.js +0 -191
  65. package/dist/commands/rules.js +0 -98
  66. package/dist/commands/scaffold-provisional-universe.js +0 -806
  67. package/dist/commands/setup.js +0 -1170
  68. package/dist/commands/skills.js +0 -447
  69. package/dist/commands/submit.js +0 -745
  70. package/dist/commands/system-map.js +0 -228
  71. package/dist/commands/triage.js +0 -598
  72. package/dist/commands/webhooks.js +0 -241
  73. package/dist/states.js +0 -46
  74. package/dist/tools/detect.js +0 -832
  75. package/dist/triage/astra.js +0 -202
  76. package/dist/triage/prompts.js +0 -188
  77. package/dist/triage/runner.js +0 -200
  78. package/dist/triage/types.js +0 -7
  79. package/dist/utils/action-output.js +0 -26
  80. package/dist/utils/analysis-api.js +0 -416
  81. package/dist/utils/design-verifier-api.js +0 -294
  82. package/dist/utils/design-verifier-history.js +0 -79
  83. package/dist/utils/design-verifier-result.js +0 -424
  84. package/dist/utils/github-api.js +0 -324
  85. package/dist/utils/pending-state.js +0 -86
  86. package/dist/utils/pr-ref.js +0 -56
  87. package/dist/utils/prompter.js +0 -328
  88. package/schemas/action.v1.json +0 -22
  89. package/schemas/ask.v1.json +0 -40
  90. package/schemas/inbox.v1.json +0 -27
  91. package/schemas/pr-status.v1.json +0 -61
  92. package/schemas/pr.v1.json +0 -97
  93. package/schemas/pr.v3.json +0 -45
  94. package/schemas/setup.v1.json +0 -75
  95. package/schemas/submit.v1.json +0 -90
  96. package/schemas/triage.v1.json +0 -103
  97. package/schemas/triage.v2.json +0 -64
@@ -1,4060 +0,0 @@
1
- import { createHash } from 'node:crypto';
2
- import { execFileSync } from 'node:child_process';
3
- import { existsSync, readFileSync, statSync } from 'node:fs';
4
- import { basename, extname, join, posix, resolve } from 'node:path';
5
- import fg from 'fast-glob';
6
- import { findGitRoot } from '../utils/hooks.js';
7
- import { scanPascalSelectorIdentities } from './cloud-verifier-pascal-selector-census.js';
8
- import { scanPythonManifestSelectorIdentities } from './cloud-verifier-python-manifest-selector-census.js';
9
- import { scanSpecializedOperationalIdentities } from './cloud-verifier-specialized-operational-census.js';
10
- import { detectDataStores, } from './cloud-verifier-data-store-census.js';
11
- export const IDENTITY_CENSUS_SCHEMA_VERSION = 'cloud-verifier-identity-census-v1';
12
- export const IDENTITY_CENSUS_CONTRACT_REVISION = 39;
13
- const MAX_SOURCE_BYTES = 4 * 1024 * 1024;
14
- const MAX_LOCKFILE_BYTES = 24 * 1024 * 1024;
15
- const MAX_EVIDENCE_TEXT_CHARS = 1_024;
16
- const MAX_SUPPORTING_TOKENS = 64;
17
- const LOCKFILE_NAMES = new Set([
18
- 'uv.lock',
19
- 'poetry.lock',
20
- 'pdm.lock',
21
- 'package-lock.json',
22
- 'npm-shrinkwrap.json',
23
- 'pnpm-lock.yaml',
24
- 'pnpm-lock.yml',
25
- 'yarn.lock',
26
- 'bun.lock',
27
- 'bun.lockb',
28
- 'cargo.lock',
29
- 'go.sum',
30
- 'gemfile.lock',
31
- 'composer.lock',
32
- ]);
33
- const MANIFEST_NAMES = new Set([
34
- 'pyproject.toml',
35
- 'package.json',
36
- 'cargo.toml',
37
- 'go.mod',
38
- 'gemfile',
39
- 'composer.json',
40
- 'pom.xml',
41
- 'build.gradle',
42
- 'build.gradle.kts',
43
- ]);
44
- const ALWAYS_SCAN_NAMES = new Set([
45
- ...LOCKFILE_NAMES,
46
- ...MANIFEST_NAMES,
47
- 'dockerfile',
48
- 'compose.yml',
49
- 'compose.yaml',
50
- 'docker-compose.yml',
51
- 'docker-compose.yaml',
52
- 'makefile',
53
- 'requirements.txt',
54
- 'requirements-prod.txt',
55
- 'requirements-production.txt',
56
- 'install.inf',
57
- ]);
58
- const SOURCE_EXTENSIONS = new Set([
59
- '.py', '.pyi', '.js', '.jsx', '.mjs', '.cjs', '.ts', '.tsx', '.mts', '.cts',
60
- '.go', '.rs', '.rb', '.java', '.kt', '.kts', '.cs', '.php', '.swift', '.scala',
61
- '.sh', '.bash', '.zsh', '.fish', '.ps1', '.sql', '.graphql', '.gql', '.proto',
62
- '.toml', '.yaml', '.yml', '.json', '.xml', '.ini', '.cfg', '.conf', '.properties',
63
- '.vue', '.pas', '.pp', '.inc', '.lpr', '.lpi', '.lpk', '.lfm', '.inf', '.plist',
64
- '.desktop', '.control',
65
- ]);
66
- const EXCLUDED_COMPONENTS = new Set([
67
- '.git', '.hg', '.svn', '.haystack', '.agents', '.codex', '.claude',
68
- '.venv', 'venv', 'node_modules', 'vendor', 'target',
69
- 'coverage', 'playwright-report', 'test-results', '__pycache__',
70
- 'test', 'tests', 'testapp', 'testapps', '__tests__', 'spec', 'specs', 'fixtures', '__fixtures__',
71
- 'examples', 'example', 'samples', 'sample', 'benchmarks', 'benchmark', 'docs', 'doc',
72
- ]);
73
- // Repository modules often qualify a non-product purpose with the project name
74
- // (for example, "project-docs" or "project-samples"). Match those purposes as
75
- // exact structured component tokens, never as fuzzy substrings.
76
- const EXCLUDED_DIRECTORY_PURPOSE_TOKENS = new Set([
77
- 'benchmark', 'benchmarks', 'doc', 'docs', 'documentation', 'example', 'examples',
78
- 'fixture', 'fixtures', 'sample', 'samples', 'spec', 'specs', 'test', 'tests',
79
- ]);
80
- // Files the data-store detector reads: dependency manifests, service manifests,
81
- // and config that name a database/cache/queue/store. A targeted, additive pass
82
- // that does not alter the identity census's own scan surface.
83
- const DATA_STORE_CONFIG_EXTENSIONS = new Set([
84
- '.yml', '.yaml', '.toml', '.json', '.ini', '.conf', '.cfg', '.properties',
85
- '.csproj', '.fsproj', '.vbproj', '.props',
86
- ]);
87
- // Whether a file (by lowercased basename) is one the data-store detector reads.
88
- // Exported so the drift check can gate on it: a change that touches no such file
89
- // cannot alter the detected stores, so the drift check can skip it entirely.
90
- export function isDataStoreInputName(lowerName) {
91
- // A lockfile is the full transitive tree, not a store declaration, and can be
92
- // large — reading it (as a .json/.yaml) only wastes work and inflates
93
- // omissions. The detector reads the direct manifest (package.json) instead.
94
- if (LOCKFILE_NAMES.has(lowerName))
95
- return false;
96
- return MANIFEST_NAMES.has(lowerName)
97
- || lowerName === 'mix.exs'
98
- || lowerName === 'packages.config'
99
- // Django names its database/cache backends in settings.py, and Elixir names
100
- // its stores in conventional config scripts (config/runtime.exs, etc.). These
101
- // are specific, conventional basenames the config-declaration patterns target
102
- // — fed by name so the whole `.py`/`.exs` source surface stays out.
103
- || lowerName === 'settings.py'
104
- || lowerName === 'config.exs' || lowerName === 'runtime.exs'
105
- || lowerName === 'prod.exs' || lowerName === 'dev.exs' || lowerName === 'releases.exs'
106
- || lowerName.startsWith('requirements')
107
- || lowerName === 'dockerfile' || lowerName.startsWith('dockerfile.') || lowerName.endsWith('.dockerfile')
108
- || lowerName === '.env' || lowerName.startsWith('.env.')
109
- || DATA_STORE_CONFIG_EXTENSIONS.has(extname(lowerName));
110
- }
111
- export function collectDataStores(root, tracked, omissions = []) {
112
- const inputs = [];
113
- for (const path of tracked) {
114
- // Apply the census's own production-file boundary so this pass matches the
115
- // identity records. This excludes the verifier's checked-in map
116
- // (`.haystack/cloud-verifier/*.yml`, whose `image: postgres` evidence would
117
- // otherwise mint stores from the map itself), plus test/example/vendor trees
118
- // — a compose under `examples/` or `tests/` is not the app's own store. The
119
- // drift check reuses this collector, so it inherits the same boundary.
120
- if (isTestLike(path))
121
- continue;
122
- const lowerName = basename(path).toLowerCase();
123
- if (!isDataStoreInputName(lowerName))
124
- continue;
125
- const absolute = join(root, path);
126
- try {
127
- if (!existsSync(absolute) || !statSync(absolute).isFile())
128
- continue;
129
- if (statSync(absolute).size > MAX_SOURCE_BYTES) {
130
- omissions.push({ path, reason: 'data-store-input-over-size-limit' });
131
- continue;
132
- }
133
- const bytes = readFileSync(absolute);
134
- if (bytes.includes(0))
135
- continue; // skip binary files
136
- inputs.push({ path, lowerName, text: bytes.toString('utf8') });
137
- }
138
- catch {
139
- // This pass reads files the main scan may not (.env, Dockerfiles,
140
- // .csproj/.props, .ini), so its read failures are not covered by the
141
- // main omission list — record them here so the gap stays visible.
142
- omissions.push({ path, reason: 'data-store-input-unreadable' });
143
- }
144
- }
145
- return detectDataStores(inputs);
146
- }
147
- function sha256(value) {
148
- return createHash('sha256').update(value).digest('hex');
149
- }
150
- function stableId(kind, identityKey) {
151
- const slug = kind.toLowerCase().replace(/[^a-z0-9]+/g, '-').replace(/^-|-$/g, '');
152
- return `census-${slug}-${sha256(`${kind}\0${identityKey}`).slice(0, 20)}`;
153
- }
154
- function recordStableId(kind, path, owner, identity) {
155
- return stableId(kind, `${path}#${owner}:${identity}`);
156
- }
157
- function normalizePath(path) {
158
- return path.split(posix.sep).join('/').replace(/^\.\//, '');
159
- }
160
- function isTestLike(path) {
161
- const sourceComponents = normalizePath(path).split('/');
162
- const normalized = sourceComponents.join('/').toLowerCase();
163
- const name = basename(normalized);
164
- const components = normalized.split('/');
165
- const productionSourceIndex = components.findIndex((component, index) => component === 'src' && components[index + 1] === 'main');
166
- const insideProductionNamespace = (index) => productionSourceIndex >= 0 && index > productionSourceIndex + 1;
167
- if (components.some((component, index) => !insideProductionNamespace(index) && EXCLUDED_COMPONENTS.has(component)))
168
- return true;
169
- const directoryHasExcludedPurpose = sourceComponents.slice(0, -1).some((component, index) => {
170
- if (insideProductionNamespace(index))
171
- return false;
172
- const tokens = component
173
- .replace(/([a-z0-9])([A-Z])/g, '$1-$2')
174
- .toLowerCase()
175
- .split(/[^a-z0-9]+/)
176
- .filter(Boolean);
177
- return tokens.some((token) => EXCLUDED_DIRECTORY_PURPOSE_TOKENS.has(token));
178
- });
179
- if (directoryHasExcludedPurpose)
180
- return true;
181
- if (ALWAYS_SCAN_NAMES.has(name))
182
- return false;
183
- return /(?:^|[._-])(test|tests|spec|fixture|mock)(?:[._-]|$)/.test(name)
184
- || /^(?:test_|conftest\.)/.test(name)
185
- || /(?:^|\.)stories\.[^.]+$/.test(name);
186
- }
187
- function shouldScan(path) {
188
- const normalized = normalizePath(path);
189
- const lowerName = basename(normalized).toLowerCase();
190
- if (isTestLike(normalized))
191
- return false;
192
- if (ALWAYS_SCAN_NAMES.has(lowerName))
193
- return true;
194
- if (normalized.startsWith('.github/workflows/'))
195
- return true;
196
- if (normalized.startsWith('deploy/') || normalized.startsWith('deployment/') || normalized.startsWith('infra/'))
197
- return true;
198
- return SOURCE_EXTENSIONS.has(extname(lowerName));
199
- }
200
- function trackedFiles(root) {
201
- try {
202
- return execFileSync('git', ['-C', root, 'ls-files', '-z'], {
203
- encoding: 'utf8',
204
- maxBuffer: 64 * 1024 * 1024,
205
- stdio: ['ignore', 'pipe', 'ignore'],
206
- }).split('\0').filter(Boolean).map(normalizePath)
207
- .filter((path) => !path.startsWith('.haystack/'))
208
- .sort();
209
- }
210
- catch {
211
- // Same entries as glob's { dot: true, nodir: true }: every non-directory,
212
- // with symlinks listed as themselves and never followed.
213
- return fg.sync('**/*', { cwd: root, dot: true, onlyFiles: false, followSymbolicLinks: false, markDirectories: true })
214
- .filter((path) => !path.endsWith('/'))
215
- .map(normalizePath)
216
- .filter((path) => !path.startsWith('.git/') &&
217
- !path.startsWith('.haystack/') &&
218
- !path.includes('/node_modules/'))
219
- .sort();
220
- }
221
- }
222
- function gitObjectId(root, revision) {
223
- try {
224
- const value = execFileSync('git', ['-C', root, 'rev-parse', '--verify', revision], {
225
- encoding: 'utf8',
226
- maxBuffer: 1024 * 1024,
227
- stdio: ['ignore', 'pipe', 'ignore'],
228
- }).trim();
229
- return /^[0-9a-f]{40}$/.test(value) ? value : null;
230
- }
231
- catch {
232
- return null;
233
- }
234
- }
235
- function textLines(root, path) {
236
- const absolute = join(root, path);
237
- const lowerName = basename(path).toLowerCase();
238
- const maxBytes = LOCKFILE_NAMES.has(lowerName) ? MAX_LOCKFILE_BYTES : MAX_SOURCE_BYTES;
239
- const stat = statSync(absolute);
240
- if (!stat.isFile() || stat.size > maxBytes)
241
- return null;
242
- const bytes = readFileSync(absolute);
243
- if (bytes.includes(0))
244
- return null;
245
- return bytes.toString('utf8').split(/\r?\n/);
246
- }
247
- // Compatibility for typed records introduced before proof_status became part
248
- // of the census contract. New scanners must set proofStatus explicitly; this
249
- // set only preserves the already-structured rev35/rev36 emitters.
250
- const LEGACY_PROOF_COMPLETE_KINDS = new Set([
251
- 'locked-production-package-artifact-state',
252
- 'locked-production-package-artifact-install-operation',
253
- 'python-installation-profile-state',
254
- 'python-requirements-installation-profile-state',
255
- 'python-requirement-variant-state',
256
- 'python-requirements-profile-install-operation',
257
- 'exact-operation-matrix-member',
258
- 'maven-product-distribution-state',
259
- 'maven-installation-profile-state',
260
- 'maven-dependency-requirement-state',
261
- 'maven-plugin-dependency-requirement-state',
262
- 'maven-build-plugin-requirement-state',
263
- 'maven-plugin-execution-entrypoint-state',
264
- 'browser-keyed-storage-state',
265
- 'workflow-publication-stage-state',
266
- 'workflow-product-publication-runtime-unit',
267
- 'workflow-publication-secret-read-operation',
268
- 'workflow-publication-command-operation',
269
- 'workflow-publication-trigger-operation',
270
- 'workflow-publication-action-operation',
271
- 'workflow-publication-host-allocation-operation',
272
- 'browser-mounted-application-role',
273
- 'browser-supervised-subscription-role',
274
- 'spring-managed-lifecycle-role',
275
- 'java-annotated-lifecycle-role',
276
- 'java-supervised-worker-role',
277
- 'spring-static-resource-inbound-operation',
278
- 'spring-controller-inbound-operation',
279
- 'vue-local-component-family',
280
- 'vue-local-directive-family',
281
- 'vue-global-component-family',
282
- 'vue-global-directive-family',
283
- 'vue-selector-entry',
284
- 'vue-selector-edge',
285
- 'java-annotated-command-family',
286
- 'java-annotated-command-entry',
287
- 'java-json-subtype-family',
288
- 'java-json-subtype-entry',
289
- 'java-deserializer-family',
290
- 'java-deserializer-entry',
291
- 'java-spring-bean-alternative-family',
292
- 'java-spring-bean-alternative-entry',
293
- 'java-selector-edge',
294
- ]);
295
- function addRecord(records, input) {
296
- const identityKey = `${input.path}#${input.owner}:${input.identity}`;
297
- const id = stableId(input.kind, identityKey);
298
- if (records.has(id))
299
- return;
300
- const proofStatus = input.proofStatus ?? (LEGACY_PROOF_COMPLETE_KINDS.has(input.kind)
301
- ? 'proof-complete'
302
- : input.presumptiveInventory ? 'discovery-lead' : 'non-inventory');
303
- // Proof-complete records are mandatory finals. Operational discovery leads
304
- // are also presumptive inventory: every exact source observation must be
305
- // resolved to a mapped identity or an evidenced exclusion instead of
306
- // disappearing behind an inventory-complete receipt. Registry-only leads
307
- // remain in the selector-surface census until structured proof upgrades them.
308
- const presumptiveInventory = proofStatus === 'proof-complete' || (proofStatus === 'discovery-lead' &&
309
- input.projections.some((projection) => projection === 'runtime_roles' ||
310
- projection === 'stateful_resources' ||
311
- projection === 'external_dependencies'));
312
- const rawIdentityComponents = {
313
- ...(input.identityComponents ?? {}),
314
- };
315
- if (proofStatus === 'proof-complete') {
316
- rawIdentityComponents.identity_kind ??= input.kind;
317
- rawIdentityComponents.production_provenance ??= `repository-source:${input.path}`;
318
- rawIdentityComponents.owner_identity ??= `${input.path}#${input.owner}`;
319
- if (input.projections.includes('registry_families')) {
320
- rawIdentityComponents.selector_domain ??= input.identity;
321
- }
322
- if (input.projections.includes('registry_entries')) {
323
- if (input.selectorKey !== undefined)
324
- rawIdentityComponents.selector_key ??= input.selectorKey;
325
- if (input.implementationKey !== undefined) {
326
- rawIdentityComponents.implementation_identity ??= input.implementationKey;
327
- }
328
- }
329
- if (input.projections.includes('registry_edges')) {
330
- if (input.parentSelectorEntryRecordId !== undefined) {
331
- rawIdentityComponents.parent_entry_identity ??= input.parentSelectorEntryRecordId;
332
- }
333
- if (input.childSelectorFamilyRecordId !== undefined) {
334
- rawIdentityComponents.child_family_identity ??= input.childSelectorFamilyRecordId;
335
- }
336
- if (input.relationKind !== undefined)
337
- rawIdentityComponents.relation_kind ??= input.relationKind;
338
- }
339
- if (input.projections.includes('runtime_roles')) {
340
- rawIdentityComponents.controller_kind ??= input.kind.startsWith('workflow-')
341
- ? 'workflow'
342
- : input.kind.startsWith('browser-') ? 'browser-host'
343
- : input.kind.startsWith('spring-') ? 'spring-container'
344
- : input.kind.startsWith('java-') ? 'jvm-application'
345
- : 'source-declared';
346
- rawIdentityComponents.entrypoint_identity ??= input.identity;
347
- rawIdentityComponents.lifecycle_kind ??= input.operationToken ?? input.kind;
348
- if (input.kind === 'workflow-product-publication-runtime-unit') {
349
- rawIdentityComponents.production_operational_unit ??= 'true';
350
- rawIdentityComponents.production_effect_kind ??= 'deploy';
351
- }
352
- }
353
- if (input.projections.includes('stateful_resources')) {
354
- rawIdentityComponents.address_grammar ??=
355
- input.addressToken ?? input.materializationRootIdentityKey ??
356
- input.packageArtifactUrl ?? input.packageArtifactFilename ?? input.identity;
357
- rawIdentityComponents.reproduction_operation ??=
358
- input.reproductionOperationToken ?? input.operationToken ?? `manifest-derived:${input.kind}`;
359
- }
360
- if (input.projections.includes('external_dependencies')) {
361
- rawIdentityComponents.boundary_kind ??= input.kind;
362
- rawIdentityComponents.direction ??= input.kind.includes('inbound') ? 'inbound' : 'outbound';
363
- rawIdentityComponents.operation_identity ??=
364
- input.operationIdentityKey ?? input.operationToken ?? input.identity;
365
- rawIdentityComponents.target_identity ??=
366
- input.addressToken ?? input.packageArtifactUrl ?? input.identity;
367
- rawIdentityComponents.object_identity ??=
368
- input.packageArtifactFilename ?? input.identity;
369
- rawIdentityComponents.flow_stage ??= input.operationToken ?? input.kind;
370
- }
371
- }
372
- const identityComponents = Object.keys(rawIdentityComponents).length === 0
373
- ? undefined
374
- : Object.fromEntries(Object.entries(rawIdentityComponents)
375
- .sort(([left], [right]) => left.localeCompare(right))
376
- .map(([key, value]) => [key, Array.isArray(value) ? [...value] : value]));
377
- const missingProofComponents = input.missingProofComponents ?? (proofStatus === 'proof-complete' ? [] : undefined);
378
- records.set(id, {
379
- id,
380
- kind: input.kind,
381
- projections: [...new Set(input.projections)].sort(),
382
- presumptive_inventory: presumptiveInventory,
383
- proof_status: proofStatus,
384
- ...(proofStatus !== 'proof-complete' ? {} : {
385
- semantic_identity_id: stableId('semantic-identity-v1', input.semanticIdentityKey ?? identityKey),
386
- }),
387
- ...(input.executionIdentityKey === undefined ? {} : {
388
- execution_identity_id: stableId('execution-identity-v1', input.executionIdentityKey),
389
- }),
390
- ...(identityComponents === undefined ? {} : { identity_components: identityComponents }),
391
- ...(missingProofComponents === undefined ? {} : {
392
- missing_proof_components: [...new Set(missingProofComponents)].sort(),
393
- }),
394
- owner_identity_key: `${input.path}#${input.owner}`,
395
- identity_key: identityKey,
396
- ...(input.selectorKey === undefined ? {} : { selector_key: input.selectorKey }),
397
- ...(input.implementationKey === undefined ? {} : { implementation_key: input.implementationKey }),
398
- ...(input.operationToken === undefined ? {} : { operation_token: input.operationToken }),
399
- ...(input.addressToken === undefined ? {} : { address_token: input.addressToken }),
400
- ...(input.packageName === undefined ? {} : { package_name: input.packageName }),
401
- ...(input.packageVersion === undefined ? {} : { package_version: input.packageVersion }),
402
- ...(input.packageArtifactKind === undefined ? {} : { package_artifact_kind: input.packageArtifactKind }),
403
- ...(input.packageArtifactFilename === undefined ? {} : { package_artifact_filename: input.packageArtifactFilename }),
404
- ...(input.packageArtifactUrl === undefined ? {} : { package_artifact_url: input.packageArtifactUrl }),
405
- ...(input.packageArtifactHash === undefined ? {} : { package_artifact_hash: input.packageArtifactHash }),
406
- ...(input.operationIdentityKey === undefined ? {} : { operation_identity_key: input.operationIdentityKey }),
407
- ...(input.materializationRootIdentityKey === undefined ? {} : {
408
- materialization_root_identity_key: input.materializationRootIdentityKey,
409
- }),
410
- ...(input.reproductionOperationToken === undefined ? {} : {
411
- reproduction_operation_token: input.reproductionOperationToken,
412
- }),
413
- ...(input.matrixAxes === undefined ? {} : {
414
- matrix_axes: input.matrixAxes.map((axis) => ({
415
- ...axis,
416
- source: {
417
- ...axis.source,
418
- anchor: boundedEvidenceText(axis.source.anchor),
419
- },
420
- })),
421
- }),
422
- ...(input.selectorFamilyRecordId === undefined ? {} : {
423
- selector_family_record_id: input.selectorFamilyRecordId,
424
- }),
425
- ...(input.parentSelectorEntryRecordId === undefined ? {} : {
426
- parent_selector_entry_record_id: input.parentSelectorEntryRecordId,
427
- }),
428
- ...(input.childSelectorFamilyRecordId === undefined ? {} : {
429
- child_selector_family_record_id: input.childSelectorFamilyRecordId,
430
- }),
431
- ...(input.relationKind === undefined ? {} : { relation_kind: input.relationKind }),
432
- source: {
433
- path: input.path,
434
- start_line: input.line,
435
- end_line: input.endLine ?? input.line,
436
- anchor: boundedEvidenceText(input.anchor),
437
- },
438
- supporting_tokens: [...new Set(input.tokens.filter(Boolean))]
439
- .slice(0, MAX_SUPPORTING_TOKENS)
440
- .map(boundedEvidenceText),
441
- });
442
- }
443
- function boundedEvidenceText(value) {
444
- if (value.length <= MAX_EVIDENCE_TEXT_CHARS)
445
- return value;
446
- const digest = createHash('sha256').update(value).digest('hex');
447
- const retained = Math.floor((MAX_EVIDENCE_TEXT_CHARS - digest.length - 32) / 2);
448
- return `${value.slice(0, retained)} … [sha256:${digest}] … ${value.slice(-retained)}`;
449
- }
450
- function indentation(line) {
451
- const prefix = line.match(/^\s*/)?.[0] ?? '';
452
- return [...prefix].reduce((total, char) => total + (char === '\t' ? 8 : 1), 0);
453
- }
454
- function pythonOwners(path, lines) {
455
- const owners = [];
456
- const stack = [];
457
- lines.forEach((line, index) => {
458
- const trimmed = line.trim();
459
- const indent = indentation(line);
460
- const continuingSignature = stack.at(-1)?.signatureDepth ?? 0;
461
- if (continuingSignature === 0 &&
462
- trimmed &&
463
- !trimmed.startsWith('#') &&
464
- !trimmed.startsWith('@')) {
465
- while (stack.length > 0 && indent <= stack[stack.length - 1].indent)
466
- stack.pop();
467
- }
468
- const match = line.match(/^\s*(?:async\s+def|def|class)\s+([A-Za-z_][A-Za-z0-9_]*)/);
469
- if (match)
470
- stack.push({
471
- indent,
472
- name: match[1],
473
- signatureDepth: Math.max(0, parenthesisDelta(line.slice(match.index ?? 0))),
474
- });
475
- else if (stack.length > 0 && stack[stack.length - 1].signatureDepth > 0) {
476
- stack[stack.length - 1].signatureDepth = Math.max(0, stack[stack.length - 1].signatureDepth + parenthesisDelta(line));
477
- }
478
- owners[index] = stack.length > 0 ? stack.map((entry) => entry.name).join('.') : basename(path);
479
- });
480
- return owners;
481
- }
482
- function collectionBracketDelta(value) {
483
- let delta = 0;
484
- let quote = null;
485
- let escaped = false;
486
- for (const char of value) {
487
- if (escaped) {
488
- escaped = false;
489
- continue;
490
- }
491
- if (char === '\\') {
492
- escaped = true;
493
- continue;
494
- }
495
- if (quote) {
496
- if (char === quote)
497
- quote = null;
498
- continue;
499
- }
500
- if (char === "'" || char === '"') {
501
- quote = char;
502
- continue;
503
- }
504
- if (char === '[' || char === '(' || char === '{')
505
- delta += 1;
506
- if (char === ']' || char === ')' || char === '}')
507
- delta -= 1;
508
- }
509
- return delta;
510
- }
511
- function pythonModuleName(path) {
512
- const withoutExtension = normalizePath(path).replace(/\.pyi?$/, '');
513
- const withoutSourceRoot = withoutExtension.startsWith('src/')
514
- ? withoutExtension.slice(4)
515
- : withoutExtension;
516
- return withoutSourceRoot.replace(/\/__init__$/, '').replaceAll('/', '.');
517
- }
518
- function stripPythonComment(value) {
519
- let quote = null;
520
- let escaped = false;
521
- for (let index = 0; index < value.length; index += 1) {
522
- const char = value[index];
523
- if (escaped) {
524
- escaped = false;
525
- continue;
526
- }
527
- if (char === '\\') {
528
- escaped = true;
529
- continue;
530
- }
531
- if (quote) {
532
- if (char === quote)
533
- quote = null;
534
- continue;
535
- }
536
- if (char === "'" || char === '"') {
537
- quote = char;
538
- continue;
539
- }
540
- if (char === '#')
541
- return value.slice(0, index);
542
- }
543
- return value;
544
- }
545
- function parsePythonLiteralCollections(path, lines) {
546
- const collections = new Map();
547
- for (let index = 0; index < lines.length; index += 1) {
548
- const assignment = lines[index].match(/^([A-Z][A-Z0-9_]*)\s*(?::[^=]+)?=\s*(\[|\()/);
549
- if (!assignment)
550
- continue;
551
- const start = index;
552
- let end = index;
553
- let depth = collectionBracketDelta(lines[index].slice(lines[index].indexOf(assignment[2])));
554
- while (depth > 0 && end + 1 < lines.length && end - start < 10_000) {
555
- end += 1;
556
- depth += collectionBracketDelta(lines[end]);
557
- }
558
- if (depth !== 0)
559
- continue;
560
- const symbol = assignment[1];
561
- const symbolIdentityKey = `${path}#${symbol}`;
562
- const items = [];
563
- for (let itemLine = start; itemLine <= end; itemLine += 1) {
564
- const text = stripPythonComment(itemLine === start
565
- ? lines[itemLine].slice(lines[itemLine].indexOf(assignment[2]) + 1)
566
- : lines[itemLine]);
567
- for (const match of text.matchAll(/(['"])(.*?)\1|\b-?\d+(?:\.\d+)?\b/g)) {
568
- const value = match[2] ?? match[0];
569
- const ordinal = items.length;
570
- items.push({
571
- value,
572
- identityKey: `${symbolIdentityKey}[${ordinal}]`,
573
- source: {
574
- path,
575
- start_line: itemLine + 1,
576
- end_line: itemLine + 1,
577
- anchor: lines[itemLine],
578
- },
579
- });
580
- }
581
- }
582
- if (items.length === 0)
583
- continue;
584
- collections.set(symbol, {
585
- symbol,
586
- symbolIdentityKey,
587
- source: {
588
- path,
589
- start_line: start + 1,
590
- end_line: end + 1,
591
- anchor: lines[start],
592
- },
593
- items,
594
- });
595
- index = end;
596
- }
597
- return collections;
598
- }
599
- function bindPythonLiteralCollections(path, lines, ownCollections, collectionsByModule) {
600
- const bindings = new Map(ownCollections);
601
- for (const line of lines) {
602
- const fromImport = line.match(/^\s*from\s+([A-Za-z_][A-Za-z0-9_.]*)\s+import\s+(.+)$/);
603
- if (fromImport) {
604
- const imported = collectionsByModule.get(fromImport[1]);
605
- if (!imported)
606
- continue;
607
- for (const rawSymbol of fromImport[2].replace(/[()]/g, '').split(',')) {
608
- const symbol = rawSymbol.trim().match(/^([A-Za-z_][A-Za-z0-9_]*)(?:\s+as\s+([A-Za-z_][A-Za-z0-9_]*))?/);
609
- if (!symbol)
610
- continue;
611
- const collection = imported.get(symbol[1]);
612
- if (collection)
613
- bindings.set(symbol[2] ?? symbol[1], collection);
614
- }
615
- continue;
616
- }
617
- const moduleImport = line.match(/^\s*import\s+([A-Za-z_][A-Za-z0-9_.]*)(?:\s+as\s+([A-Za-z_][A-Za-z0-9_]*))?/);
618
- if (!moduleImport)
619
- continue;
620
- const imported = collectionsByModule.get(moduleImport[1]);
621
- if (!imported)
622
- continue;
623
- const prefix = moduleImport[2] ?? moduleImport[1].split('.').at(-1);
624
- for (const [symbol, collection] of imported) {
625
- bindings.set(`${prefix}.${symbol}`, collection);
626
- }
627
- }
628
- return bindings;
629
- }
630
- function pythonFunctionDefinitions(lines) {
631
- const functions = new Map();
632
- for (let index = 0; index < lines.length; index += 1) {
633
- const match = lines[index].match(/^(\s*)(?:async\s+def|def)\s+([A-Za-z_][A-Za-z0-9_]*)\s*\(/);
634
- if (!match)
635
- continue;
636
- const indent = indentation(match[1]);
637
- let signatureEnd = index;
638
- let signatureDepth = parenthesisDelta(lines[index].slice(lines[index].indexOf('(')));
639
- while (signatureDepth > 0 && signatureEnd + 1 < lines.length) {
640
- signatureEnd += 1;
641
- signatureDepth += parenthesisDelta(lines[signatureEnd]);
642
- }
643
- let end = lines.length;
644
- for (let cursor = signatureEnd + 1; cursor < lines.length; cursor += 1) {
645
- const trimmed = lines[cursor].trim();
646
- if (trimmed && !trimmed.startsWith('#') && indentation(lines[cursor]) <= indent) {
647
- end = cursor;
648
- break;
649
- }
650
- }
651
- functions.set(match[2], {
652
- name: match[2],
653
- startLine: index + 1,
654
- endLine: end,
655
- indent,
656
- });
657
- }
658
- return functions;
659
- }
660
- function mergePythonAxes(...groups) {
661
- const byIdentity = new Map();
662
- for (const group of groups) {
663
- for (const collection of group)
664
- byIdentity.set(collection.symbolIdentityKey, collection);
665
- }
666
- return [...byIdentity.values()].sort((left, right) => left.symbolIdentityKey.localeCompare(right.symbolIdentityKey));
667
- }
668
- function pythonAxisSetKey(axes) {
669
- return axes.map((axis) => axis.symbolIdentityKey).sort().join('\0');
670
- }
671
- function analyzePythonOperationMatrices(path, lines, collectionBindings) {
672
- const functions = pythonFunctionDefinitions(lines);
673
- const loopsByFunction = new Map();
674
- const callEdges = [];
675
- const incoming = new Map([...functions.keys()].map((name) => [name, 0]));
676
- for (const definition of functions.values()) {
677
- const loops = [];
678
- for (let lineIndex = definition.startLine; lineIndex < definition.endLine; lineIndex += 1) {
679
- const line = lines[lineIndex];
680
- const loop = line.match(/^\s*(?:async\s+)?for\s+.+?\s+in\s+(?:enumerate\(\s*)?([A-Za-z_][A-Za-z0-9_.]*)(?:\s*\))?\s*:/);
681
- if (loop) {
682
- const collection = collectionBindings.get(loop[1]);
683
- if (collection) {
684
- const loopIndent = indentation(line);
685
- let loopEnd = definition.endLine;
686
- for (let cursor = lineIndex + 1; cursor < definition.endLine; cursor += 1) {
687
- const current = lines[cursor].trim();
688
- if (current && !current.startsWith('#') && indentation(lines[cursor]) <= loopIndent) {
689
- loopEnd = cursor;
690
- break;
691
- }
692
- }
693
- loops.push({ startLine: lineIndex + 1, endLine: loopEnd, collection });
694
- }
695
- }
696
- for (const callee of functions.keys()) {
697
- if (lineIndex + 1 === functions.get(callee)?.startLine)
698
- continue;
699
- const escaped = callee.replace(/[.*+?^${}()|[\]\\]/g, '\\$&');
700
- if (!new RegExp(`\\b${escaped}\\s*\\(`).test(line))
701
- continue;
702
- const loopAxes = loops
703
- .filter((candidate) => candidate.startLine <= lineIndex + 1 && lineIndex + 1 <= candidate.endLine)
704
- .map((candidate) => candidate.collection);
705
- const argumentAxes = [...collectionBindings.entries()]
706
- .filter(([binding]) => {
707
- const escapedBinding = binding.replace(/[.*+?^${}()|[\]\\]/g, '\\$&');
708
- return new RegExp(`\\b${escapedBinding}\\b`).test(line);
709
- })
710
- .map(([, collection]) => collection);
711
- callEdges.push({
712
- caller: definition.name,
713
- callee,
714
- line: lineIndex + 1,
715
- axes: mergePythonAxes(loopAxes, argumentAxes),
716
- });
717
- incoming.set(callee, (incoming.get(callee) ?? 0) + 1);
718
- }
719
- }
720
- loopsByFunction.set(definition.name, loops);
721
- }
722
- const contextsByFunction = new Map();
723
- for (const name of functions.keys()) {
724
- contextsByFunction.set(name, incoming.get(name) === 0 ? [[]] : []);
725
- }
726
- let changed = true;
727
- for (let pass = 0; changed && pass < functions.size * 4 + 4; pass += 1) {
728
- changed = false;
729
- for (const edge of callEdges) {
730
- const callerContexts = contextsByFunction.get(edge.caller) ?? [];
731
- const calleeContexts = contextsByFunction.get(edge.callee) ?? [];
732
- const existing = new Set(calleeContexts.map(pythonAxisSetKey));
733
- for (const callerContext of callerContexts) {
734
- const context = mergePythonAxes(callerContext, edge.axes);
735
- const key = pythonAxisSetKey(context);
736
- if (existing.has(key))
737
- continue;
738
- calleeContexts.push(context);
739
- existing.add(key);
740
- changed = true;
741
- }
742
- contextsByFunction.set(edge.callee, calleeContexts);
743
- }
744
- }
745
- return { path, lines, functions, loopsByFunction, contextsByFunction };
746
- }
747
- function scanPythonExecutableEntrypoint(path, lines, records) {
748
- const functions = pythonFunctionDefinitions(lines);
749
- for (let index = 0; index < lines.length; index += 1) {
750
- const guard = lines[index].match(/^\s*if\s+(?:__name__\s*==\s*(['"])__main__\1|(['"])__main__\2\s*==\s*__name__)\s*:/);
751
- if (!guard)
752
- continue;
753
- const guardIndent = indentation(lines[index]);
754
- let end = lines.length;
755
- for (let cursor = index + 1; cursor < lines.length; cursor += 1) {
756
- const trimmed = lines[cursor].trim();
757
- if (trimmed && !trimmed.startsWith('#') && indentation(lines[cursor]) <= guardIndent) {
758
- end = cursor;
759
- break;
760
- }
761
- }
762
- const implementations = [...functions.keys()].filter((name) => {
763
- const escaped = name.replace(/[.*+?^${}()|[\]\\]/g, '\\$&');
764
- return lines.slice(index + 1, end).some((line) => new RegExp(`\\b${escaped}\\s*\\(`).test(line));
765
- });
766
- addRecord(records, {
767
- kind: 'python-executable-entrypoint-role',
768
- projections: ['runtime_roles'],
769
- presumptiveInventory: true,
770
- path,
771
- line: index + 1,
772
- anchor: lines[index],
773
- owner: '__main__',
774
- identity: `python-script:${path}:__main__`,
775
- tokens: [path, '__main__', ...implementations],
776
- implementationKey: implementations.join('|') || 'module-body',
777
- operationToken: 'execute-python-script',
778
- addressToken: path,
779
- operationIdentityKey: `${path}#__main__`,
780
- });
781
- }
782
- }
783
- function scanPythonEmbeddedConfiguration(path, lines, collections, importedCollectionIdentityKeys, records) {
784
- for (const collection of collections.values()) {
785
- const used = importedCollectionIdentityKeys.has(collection.symbolIdentityKey) || lines.some((line, index) => index + 1 !== collection.source.start_line &&
786
- new RegExp(`\\b${collection.symbol}\\b`).test(line));
787
- if (!used)
788
- continue;
789
- addRecord(records, {
790
- kind: 'python-embedded-configuration-state',
791
- projections: ['stateful_resources'],
792
- presumptiveInventory: true,
793
- path,
794
- line: collection.source.start_line,
795
- anchor: collection.source.anchor,
796
- owner: 'module-configuration',
797
- identity: `embedded-configuration:${collection.symbolIdentityKey}`,
798
- tokens: [collection.symbol, ...collection.items.map((item) => item.value)],
799
- addressToken: collection.symbol,
800
- materializationRootIdentityKey: collection.symbolIdentityKey,
801
- });
802
- }
803
- lines.forEach((line, index) => {
804
- const template = line.match(/^([A-Z][A-Z0-9_]*)\s*(?::[^=]+)?=\s*(?:[rubfRUBF]*)?(?:'''|""")/);
805
- if (!template)
806
- return;
807
- const symbol = template[1];
808
- const used = lines.some((candidate, candidateIndex) => candidateIndex !== index && new RegExp(`\\b${symbol}\\b`).test(candidate));
809
- if (!used)
810
- return;
811
- addRecord(records, {
812
- kind: 'python-embedded-template-state',
813
- projections: ['stateful_resources'],
814
- presumptiveInventory: true,
815
- path,
816
- line: index + 1,
817
- anchor: line,
818
- owner: 'module-configuration',
819
- identity: `embedded-template:${path}#${symbol}`,
820
- tokens: [symbol],
821
- addressToken: symbol,
822
- materializationRootIdentityKey: `${path}#${symbol}`,
823
- });
824
- });
825
- }
826
- const EXACT_MATRIX_OPERATION_KINDS = new Set([
827
- 'client-boundary-operation',
828
- 'http-operation',
829
- 'http-wrapper-operation',
830
- 'imported-sdk-instance-operation',
831
- 'socket-operation',
832
- ]);
833
- function cartesianMatrixAxes(axes, limit = 25_000) {
834
- const cardinality = axes.reduce((total, axis) => total * axis.items.length, 1);
835
- if (!Number.isSafeInteger(cardinality) || cardinality > limit)
836
- return null;
837
- let rows = [[]];
838
- for (const axis of axes) {
839
- const next = [];
840
- for (const row of rows) {
841
- for (const item of axis.items) {
842
- next.push([...row, {
843
- symbol_identity_key: axis.symbolIdentityKey,
844
- item_identity_key: item.identityKey,
845
- item_value: item.value,
846
- source: item.source,
847
- }]);
848
- }
849
- }
850
- rows = next;
851
- }
852
- return rows;
853
- }
854
- function emitPythonExactOperationMatrices(analysis, records) {
855
- const groups = new Map();
856
- for (const record of records.values()) {
857
- if (record.source.path !== analysis.path ||
858
- !EXACT_MATRIX_OPERATION_KINDS.has(record.kind) ||
859
- !record.projections.includes('external_dependencies') ||
860
- !record.operation_token)
861
- continue;
862
- const owner = record.owner_identity_key.slice(record.owner_identity_key.indexOf('#') + 1);
863
- if (!analysis.functions.has(owner))
864
- continue;
865
- const key = `${owner}\0${record.source.start_line}\0${record.operation_token}`;
866
- const group = groups.get(key) ?? [];
867
- group.push(record);
868
- groups.set(key, group);
869
- }
870
- for (const group of groups.values()) {
871
- const representative = [...group].sort((left, right) => Number(right.presumptive_inventory) - Number(left.presumptive_inventory) ||
872
- left.kind.localeCompare(right.kind))[0];
873
- const owner = representative.owner_identity_key.slice(representative.owner_identity_key.indexOf('#') + 1);
874
- const contexts = analysis.contextsByFunction.get(owner) ?? [];
875
- const localAxes = (analysis.loopsByFunction.get(owner) ?? [])
876
- .filter((loop) => loop.startLine <= representative.source.start_line &&
877
- representative.source.start_line <= loop.endLine)
878
- .map((loop) => loop.collection);
879
- const axisSets = new Map();
880
- let hasUnexpandedInvocation = false;
881
- for (const context of contexts) {
882
- const axes = mergePythonAxes(context, localAxes);
883
- if (axes.length === 0) {
884
- hasUnexpandedInvocation = true;
885
- continue;
886
- }
887
- axisSets.set(pythonAxisSetKey(axes), axes);
888
- }
889
- if (axisSets.size === 0)
890
- continue;
891
- const operationToken = representative.operation_token;
892
- const callsiteIdentity = `${analysis.path}#${owner}:${operationToken}` +
893
- `@${representative.source.start_line}`;
894
- for (const axes of axisSets.values()) {
895
- const matrixRows = cartesianMatrixAxes(axes);
896
- if (!matrixRows) {
897
- hasUnexpandedInvocation = true;
898
- continue;
899
- }
900
- for (const matrixAxes of matrixRows) {
901
- const itemIdentity = matrixAxes.map((axis) => axis.item_identity_key).join('|');
902
- const operationIdentityKey = `${callsiteIdentity}:${itemIdentity}`;
903
- addRecord(records, {
904
- kind: 'exact-operation-matrix-member',
905
- projections: ['external_dependencies'],
906
- presumptiveInventory: true,
907
- path: analysis.path,
908
- line: representative.source.start_line,
909
- anchor: representative.source.anchor,
910
- owner,
911
- identity: operationIdentityKey,
912
- tokens: [
913
- operationToken,
914
- ...matrixAxes.map((axis) => axis.item_value),
915
- ],
916
- operationToken,
917
- addressToken: matrixAxes.map((axis) => axis.item_value).join('|'),
918
- operationIdentityKey,
919
- matrixAxes,
920
- });
921
- }
922
- }
923
- if (!hasUnexpandedInvocation) {
924
- for (const record of group)
925
- record.presumptive_inventory = false;
926
- }
927
- }
928
- }
929
- function executableLineMask(path, lines) {
930
- const extension = extname(path.toLowerCase());
931
- if (extension === '.pyi' || path.endsWith('.d.ts'))
932
- return lines.map(() => false);
933
- if (extension === '.vue') {
934
- let inScript = false;
935
- return lines.map((line) => {
936
- if (/<script(?:\s[^>]*)?>/i.test(line)) {
937
- inScript = true;
938
- return false;
939
- }
940
- if (/<\/script>/i.test(line)) {
941
- inScript = false;
942
- return false;
943
- }
944
- const trimmed = line.trim();
945
- return inScript && Boolean(trimmed) &&
946
- !trimmed.startsWith('//') && !trimmed.startsWith('/*') && !trimmed.startsWith('*');
947
- });
948
- }
949
- if (extension !== '.py') {
950
- return lines.map((line) => {
951
- const trimmed = line.trim();
952
- return Boolean(trimmed) && !trimmed.startsWith('//') && !trimmed.startsWith('/*') && !trimmed.startsWith('*');
953
- });
954
- }
955
- let delimiter = null;
956
- return lines.map((line) => {
957
- const trimmed = line.trim();
958
- if (delimiter) {
959
- const end = line.indexOf(delimiter);
960
- if (end >= 0)
961
- delimiter = null;
962
- return false;
963
- }
964
- if (!trimmed || trimmed.startsWith('#'))
965
- return false;
966
- const single = line.indexOf("'''");
967
- const double = line.indexOf('"""');
968
- const start = single < 0 ? double : (double < 0 ? single : Math.min(single, double));
969
- if (start < 0)
970
- return true;
971
- const selected = start === single ? "'''" : '"""';
972
- if (line.indexOf(selected, start + 3) < 0)
973
- delimiter = selected;
974
- return start > 0 && Boolean(line.slice(0, start).trim());
975
- });
976
- }
977
- function literalTokens(value) {
978
- const tokens = [];
979
- for (const match of value.matchAll(/(['"])(.*?)\1|\b(?:true|false|null|none)\b|\b-?\d+(?:\.\d+)?\b/gi)) {
980
- const token = match[2] ?? match[0];
981
- if (!tokens.includes(token))
982
- tokens.push(token);
983
- }
984
- return tokens;
985
- }
986
- function selectorProjections(base, selectorSurface) {
987
- const projections = [base];
988
- if (/(?:^|[^a-z])(?:api|backend|callback|client|command|endpoint|event|export|format|grant|handler|hook|http|integration|listener|message|method|operation|provider|protocol|request|response|route|sdk|signal|stream|transport|url)(?:[^a-z]|$)/i
989
- .test(selectorSurface)) {
990
- projections.push('external_dependencies');
991
- }
992
- if (/(?:^|[^a-z])(?:cache|caches|callback|callbacks|config|configs|configuration|credential|credentials|hook|hooks|installed|patch|patches|plugin|plugins|queue|queues|registry|registries|session|sessions|state|states|store|stores|token|tokens|variable|variables|wrapper|wrappers)(?:[^a-z]|$)/i
993
- .test(selectorSurface)) {
994
- projections.push('stateful_resources');
995
- }
996
- return projections;
997
- }
998
- function scanPythonSelectors(path, lines, owners, executable, records) {
999
- const activeChains = new Map();
1000
- lines.forEach((line, index) => {
1001
- if (!executable[index])
1002
- return;
1003
- const lineNumber = index + 1;
1004
- const indent = indentation(line);
1005
- const branch = line.match(/^\s*(if|elif)\s+(.+):\s*(?:#.*)?$/);
1006
- if (branch) {
1007
- const owner = owners[index];
1008
- const prior = activeChains.get(indent);
1009
- const chain = branch[1] === 'elif' && prior?.owner === owner
1010
- ? prior
1011
- : { owner, line: lineNumber, condition: branch[2].trim() };
1012
- activeChains.set(indent, chain);
1013
- const selectors = literalTokens(branch[2]);
1014
- const recognizedDispatch = selectors.length > 0
1015
- || /\b(?:isinstance|issubclass|len|type)\s*\(|\.(?:startswith|endswith)\s*\(/.test(branch[2]);
1016
- if (!recognizedDispatch)
1017
- return;
1018
- const familyIdentity = `${chain.owner}:if-chain:${chain.line}`;
1019
- const familyProjections = selectorProjections('registry_families', `${chain.owner} ${chain.condition}`);
1020
- addRecord(records, {
1021
- kind: 'conditional-dispatch-family', projections: familyProjections, presumptiveInventory: true,
1022
- path, line: chain.line, anchor: lines[chain.line - 1], owner: chain.owner,
1023
- identity: familyIdentity, tokens: [chain.condition], selectorKey: branch[2].trim(),
1024
- });
1025
- (selectors.length > 0 ? selectors : [branch[2].trim()]).forEach((selector) => {
1026
- addRecord(records, {
1027
- kind: 'conditional-dispatch-entry',
1028
- projections: selectorProjections('registry_entries', `${chain.owner} ${branch[2]}`),
1029
- presumptiveInventory: true,
1030
- path, line: lineNumber, anchor: line, owner: chain.owner,
1031
- identity: `${familyIdentity}:${selector}:${lineNumber}`, tokens: [branch[2].trim(), selector],
1032
- selectorKey: selector, implementationKey: `branch@${path}:${lineNumber}`,
1033
- });
1034
- });
1035
- }
1036
- else if (line.trim() && !line.trim().startsWith('#')) {
1037
- for (const chainIndent of [...activeChains.keys()]) {
1038
- if (indent < chainIndent)
1039
- activeChains.delete(chainIndent);
1040
- }
1041
- }
1042
- const ternary = line.match(/(.+?)\s+if\s+(.+?)\s+else\s+(.+)/);
1043
- if (ternary && !/^\s*(?:if|elif)\s/.test(line)) {
1044
- const owner = owners[index];
1045
- const familyIdentity = `${owner}:conditional-expression:${lineNumber}`;
1046
- addRecord(records, {
1047
- kind: 'conditional-expression-family',
1048
- projections: selectorProjections('registry_families', `${owner} ${ternary[2]}`),
1049
- presumptiveInventory: true,
1050
- path, line: lineNumber, anchor: line, owner, identity: familyIdentity, tokens: [ternary[2].trim()],
1051
- });
1052
- [
1053
- ['true', ternary[1].trim()],
1054
- ['false', ternary[3].trim()],
1055
- ].forEach(([selector, implementation]) => addRecord(records, {
1056
- kind: 'conditional-expression-entry',
1057
- projections: selectorProjections('registry_entries', `${owner} ${ternary[2]}`),
1058
- presumptiveInventory: true,
1059
- path, line: lineNumber, anchor: line, owner,
1060
- identity: `${familyIdentity}:${selector}`, tokens: [ternary[2].trim(), implementation],
1061
- selectorKey: selector, implementationKey: implementation,
1062
- }));
1063
- }
1064
- });
1065
- for (let index = 0; index < lines.length; index += 1) {
1066
- if (!executable[index])
1067
- continue;
1068
- const assignment = lines[index].match(/^\s*([A-Za-z_][A-Za-z0-9_.]*)\s*(?::[^=]+)?=\s*\{\s*$/);
1069
- if (!assignment)
1070
- continue;
1071
- const owner = owners[index];
1072
- const baseIndent = indentation(lines[index]);
1073
- const entries = [];
1074
- let end = index + 1;
1075
- for (; end < lines.length; end += 1) {
1076
- const current = lines[end];
1077
- if (current.trim() === '}' && indentation(current) <= baseIndent)
1078
- break;
1079
- const entry = current.match(/^\s*(['"]?)([^'"\s][^:'"]*|[^'"]+)\1\s*:\s*(.+?)(?:,\s*)?(?:#.*)?$/);
1080
- if (entry)
1081
- entries.push({ line: end + 1, key: entry[2].trim(), value: entry[3].trim() });
1082
- }
1083
- if (entries.length < 2)
1084
- continue;
1085
- const familyIdentity = `${owner}:mapping:${assignment[1]}:${index + 1}`;
1086
- addRecord(records, {
1087
- kind: 'mapping-dispatch-family',
1088
- projections: selectorProjections('registry_families', `${owner} ${assignment[1]}`),
1089
- presumptiveInventory: true,
1090
- path, line: index + 1, anchor: lines[index], owner, identity: familyIdentity, tokens: [assignment[1]],
1091
- });
1092
- entries.forEach((entry) => addRecord(records, {
1093
- kind: 'mapping-dispatch-entry',
1094
- projections: selectorProjections('registry_entries', `${owner} ${assignment[1]} ${entry.key}`),
1095
- presumptiveInventory: true,
1096
- path, line: entry.line, anchor: lines[entry.line - 1], owner,
1097
- identity: `${familyIdentity}:${entry.key}`, tokens: [entry.key, entry.value],
1098
- selectorKey: entry.key, implementationKey: entry.value,
1099
- }));
1100
- index = Math.max(index, end - 1);
1101
- }
1102
- }
1103
- function scanGenericSelectors(path, lines, executable, records) {
1104
- let switchOwner = null;
1105
- lines.forEach((line, index) => {
1106
- if (!executable[index])
1107
- return;
1108
- const lineNumber = index + 1;
1109
- const switchMatch = line.match(/\bswitch\s*\((.+)\)/);
1110
- if (switchMatch) {
1111
- switchOwner = { line: lineNumber, expression: switchMatch[1].trim() };
1112
- addRecord(records, {
1113
- kind: 'switch-dispatch-family',
1114
- projections: selectorProjections('registry_families', switchMatch[1]),
1115
- presumptiveInventory: true,
1116
- path, line: lineNumber, anchor: line, owner: `switch@${lineNumber}`,
1117
- identity: `switch:${lineNumber}:${switchMatch[1].trim()}`, tokens: [switchMatch[1].trim()],
1118
- });
1119
- }
1120
- const caseMatch = line.match(/\bcase\s+([^:]+):/);
1121
- if (caseMatch && switchOwner) {
1122
- const selector = caseMatch[1].trim().replace(/^['"]|['"]$/g, '');
1123
- addRecord(records, {
1124
- kind: 'switch-dispatch-entry',
1125
- projections: selectorProjections('registry_entries', `${switchOwner.expression} ${caseMatch[1]}`),
1126
- presumptiveInventory: true,
1127
- path, line: lineNumber, anchor: line, owner: `switch@${switchOwner.line}`,
1128
- identity: `switch:${switchOwner.line}:${selector}`, tokens: [caseMatch[1].trim()],
1129
- selectorKey: selector, implementationKey: `branch@${path}:${lineNumber}`,
1130
- });
1131
- }
1132
- const conditional = line.match(/\b(?:if|else\s+if)\s*\((.+)\)/);
1133
- if (conditional) {
1134
- const selectors = literalTokens(conditional[1]);
1135
- if (selectors.length === 0 && !/\b(?:instanceof|typeof)\b|\.length\b|\.(?:startsWith|endsWith)\s*\(/.test(conditional[1]))
1136
- return;
1137
- const owner = `conditional@${lineNumber}`;
1138
- addRecord(records, {
1139
- kind: 'conditional-dispatch-family',
1140
- projections: selectorProjections('registry_families', conditional[1]),
1141
- presumptiveInventory: true,
1142
- path, line: lineNumber, anchor: line, owner, identity: `${owner}:${conditional[1].trim()}`,
1143
- tokens: [conditional[1].trim()],
1144
- });
1145
- (selectors.length > 0 ? selectors : [conditional[1].trim()]).forEach((selector) => addRecord(records, {
1146
- kind: 'conditional-dispatch-entry',
1147
- projections: selectorProjections('registry_entries', `${conditional[1]} ${selector}`),
1148
- presumptiveInventory: true,
1149
- path, line: lineNumber, anchor: line, owner, identity: `${owner}:${selector}`,
1150
- tokens: [conditional[1].trim(), selector], selectorKey: selector,
1151
- implementationKey: `branch@${path}:${lineNumber}`,
1152
- }));
1153
- }
1154
- const ternaryIndex = line.indexOf('?');
1155
- const colonIndex = ternaryIndex < 0 ? -1 : line.indexOf(':', ternaryIndex + 1);
1156
- if (ternaryIndex > 0 && colonIndex > ternaryIndex) {
1157
- const condition = line.slice(0, ternaryIndex).trim();
1158
- const owner = `conditional-expression@${lineNumber}`;
1159
- addRecord(records, {
1160
- kind: 'conditional-expression-family',
1161
- projections: selectorProjections('registry_families', condition),
1162
- presumptiveInventory: true,
1163
- path, line: lineNumber, anchor: line, owner, identity: `${owner}:${condition}`, tokens: [condition],
1164
- });
1165
- [['true', line.slice(ternaryIndex + 1, colonIndex).trim()], ['false', line.slice(colonIndex + 1).trim()]]
1166
- .forEach(([selector, implementation]) => addRecord(records, {
1167
- kind: 'conditional-expression-entry',
1168
- projections: selectorProjections('registry_entries', `${condition} ${selector}`),
1169
- presumptiveInventory: true,
1170
- path, line: lineNumber, anchor: line, owner, identity: `${owner}:${selector}`,
1171
- tokens: [condition, implementation], selectorKey: selector, implementationKey: implementation,
1172
- }));
1173
- }
1174
- });
1175
- }
1176
- function resolvedSourceImport(importerPath, specifier, extension, aliases, knownPaths) {
1177
- let joined = null;
1178
- if (specifier.startsWith('.')) {
1179
- joined = normalizePath(posix.normalize(posix.join(posix.dirname(importerPath), specifier)));
1180
- }
1181
- else {
1182
- for (const [prefix, target] of aliases) {
1183
- if (!specifier.startsWith(prefix))
1184
- continue;
1185
- joined = normalizePath(posix.normalize(posix.join(target, specifier.slice(prefix.length))));
1186
- break;
1187
- }
1188
- }
1189
- if (!joined)
1190
- return null;
1191
- const candidates = extname(joined)
1192
- ? [joined]
1193
- : [`${joined}${extension}`, `${joined}/index${extension}`];
1194
- return candidates.find((candidate) => knownPaths.has(candidate)) ?? candidates[0];
1195
- }
1196
- function scanVueSelectorCensus(root, paths, records) {
1197
- const families = [];
1198
- const knownPaths = new Set(paths.map(normalizePath));
1199
- const aliases = new Map();
1200
- for (const configPath of paths.filter((candidate) => /(?:^|\/)tsconfig(?:\.[^/]+)?\.json$/.test(candidate))) {
1201
- let configLines = null;
1202
- try {
1203
- configLines = textLines(root, configPath);
1204
- }
1205
- catch {
1206
- configLines = null;
1207
- }
1208
- if (!configLines)
1209
- continue;
1210
- const source = configLines.join('\n');
1211
- for (const alias of source.matchAll(/"([^"]*\/)?\*"\s*:\s*\[\s*"([^"]*\/)?\*"/g)) {
1212
- const prefix = alias[1] ?? '';
1213
- const target = normalizePath(posix.join(posix.dirname(configPath), alias[2] ?? ''));
1214
- if (prefix)
1215
- aliases.set(prefix, target);
1216
- }
1217
- }
1218
- for (const path of paths.filter((candidate) => candidate.endsWith('.vue') && shouldScan(candidate))) {
1219
- let lines = null;
1220
- try {
1221
- lines = textLines(root, path);
1222
- }
1223
- catch {
1224
- lines = null;
1225
- }
1226
- if (!lines)
1227
- continue;
1228
- const imports = new Map();
1229
- lines.forEach((line) => {
1230
- const imported = line.match(/^\s*import\s+([A-Za-z_$][\w$]*)\s+from\s+['"]([^'"]+)['"]/);
1231
- if (!imported)
1232
- return;
1233
- const resolved = resolvedSourceImport(path, imported[2], '.vue', aliases, knownPaths);
1234
- if (resolved)
1235
- imports.set(imported[1], resolved);
1236
- });
1237
- for (let index = 0; index < lines.length; index += 1) {
1238
- const opening = lines[index].match(/^\s*(components|directives)\s*:\s*\{\s*(.*?)\s*\}\s*,?\s*$/)
1239
- ?? lines[index].match(/^\s*(components|directives)\s*:\s*\{\s*$/);
1240
- if (!opening)
1241
- continue;
1242
- const domain = opening[1];
1243
- const entries = [];
1244
- const sameLineBody = opening[2];
1245
- if (sameLineBody) {
1246
- sameLineBody.split(',').map((value) => value.trim()).filter(Boolean).forEach((value) => {
1247
- const pair = value.match(/^(?:['"]([^'"]+)['"]|([A-Za-z_$][\w$-]*))\s*:\s*([A-Za-z_$][\w$]*)$/);
1248
- const shorthand = value.match(/^([A-Za-z_$][\w$]*)$/);
1249
- if (pair)
1250
- entries.push({
1251
- selector: pair[1] ?? pair[2], implementation: pair[3], line: index + 1, anchor: lines[index],
1252
- });
1253
- else if (shorthand)
1254
- entries.push({
1255
- selector: shorthand[1], implementation: shorthand[1], line: index + 1, anchor: lines[index],
1256
- });
1257
- });
1258
- }
1259
- else {
1260
- const baseIndent = indentation(lines[index]);
1261
- let cursor = index + 1;
1262
- for (; cursor < lines.length; cursor += 1) {
1263
- const current = lines[cursor];
1264
- if (/^\s*}\s*,?\s*$/.test(current) && indentation(current) <= baseIndent)
1265
- break;
1266
- const pair = current.match(/^\s*(?:['"]([^'"]+)['"]|([A-Za-z_$][\w$-]*))\s*:\s*([A-Za-z_$][\w$]*)\s*,?\s*$/);
1267
- const shorthand = current.match(/^\s*([A-Za-z_$][\w$]*)\s*,?\s*$/);
1268
- if (pair)
1269
- entries.push({
1270
- selector: pair[1] ?? pair[2], implementation: pair[3], line: cursor + 1, anchor: current,
1271
- });
1272
- else if (shorthand)
1273
- entries.push({
1274
- selector: shorthand[1], implementation: shorthand[1], line: cursor + 1, anchor: current,
1275
- });
1276
- }
1277
- index = cursor;
1278
- }
1279
- if (entries.length === 0)
1280
- continue;
1281
- families.push({
1282
- path,
1283
- owner: 'vue-component-options',
1284
- identity: `vue-options:${domain}`,
1285
- kind: domain === 'components' ? 'vue-local-component-family' : 'vue-local-directive-family',
1286
- line: index + 1,
1287
- anchor: lines[index] ?? `<${domain}>`,
1288
- entries,
1289
- imports,
1290
- });
1291
- }
1292
- const globalEntries = new Map();
1293
- lines.forEach((line, index) => {
1294
- for (const match of line.matchAll(/\b([A-Za-z_$][\w$]*)\.(component|directive)\(\s*['"]([^'"]+)['"]\s*,\s*([A-Za-z_$][\w$]*)/g)) {
1295
- const key = `${match[1]}.${match[2]}`;
1296
- const entries = globalEntries.get(key) ?? [];
1297
- entries.push({ selector: match[3], implementation: match[4], line: index + 1, anchor: line });
1298
- globalEntries.set(key, entries);
1299
- }
1300
- });
1301
- globalEntries.forEach((entries, key) => families.push({
1302
- path,
1303
- owner: `vue-registration:${key}`,
1304
- identity: `vue-registration:${key}`,
1305
- kind: key.endsWith('.component') ? 'vue-global-component-family' : 'vue-global-directive-family',
1306
- line: entries[0].line,
1307
- anchor: entries[0].anchor,
1308
- entries,
1309
- imports,
1310
- }));
1311
- }
1312
- const familyIdByPath = new Map();
1313
- const emittedEntries = [];
1314
- families.forEach((family) => {
1315
- const familyId = recordStableId(family.kind, family.path, family.owner, family.identity);
1316
- const pathFamilies = familyIdByPath.get(family.path) ?? [];
1317
- pathFamilies.push(familyId);
1318
- familyIdByPath.set(family.path, pathFamilies);
1319
- addRecord(records, {
1320
- kind: family.kind,
1321
- projections: ['registry_families'],
1322
- presumptiveInventory: true,
1323
- path: family.path,
1324
- line: family.line,
1325
- anchor: family.anchor,
1326
- owner: family.owner,
1327
- identity: family.identity,
1328
- tokens: [family.identity],
1329
- });
1330
- family.entries.forEach((entry) => {
1331
- const implementationPath = family.imports.get(entry.implementation);
1332
- const implementationKey = implementationPath
1333
- ? `${implementationPath}#default-export`
1334
- : `${family.path}#${entry.implementation}`;
1335
- const entryIdentity = `${family.identity}:${entry.selector}:${implementationKey}`;
1336
- const entryId = recordStableId('vue-selector-entry', family.path, family.owner, entryIdentity);
1337
- addRecord(records, {
1338
- kind: 'vue-selector-entry',
1339
- projections: ['registry_entries'],
1340
- presumptiveInventory: true,
1341
- path: family.path,
1342
- line: entry.line,
1343
- anchor: entry.anchor,
1344
- owner: family.owner,
1345
- identity: entryIdentity,
1346
- tokens: [entry.selector, entry.implementation, implementationKey],
1347
- selectorKey: entry.selector,
1348
- implementationKey,
1349
- selectorFamilyRecordId: familyId,
1350
- });
1351
- emittedEntries.push({ entryId, family, implementation: entry.implementation });
1352
- });
1353
- });
1354
- emittedEntries.forEach(({ entryId, family, implementation }) => {
1355
- const implementationPath = family.imports.get(implementation);
1356
- if (!implementationPath)
1357
- return;
1358
- const childFamilyIds = familyIdByPath.get(implementationPath) ?? [];
1359
- childFamilyIds.forEach((childFamilyId) => addRecord(records, {
1360
- kind: 'vue-selector-edge',
1361
- projections: ['registry_edges'],
1362
- presumptiveInventory: true,
1363
- path: family.path,
1364
- line: family.line,
1365
- anchor: family.anchor,
1366
- owner: family.owner,
1367
- identity: `edge:${entryId}:${childFamilyId}`,
1368
- tokens: [entryId, childFamilyId],
1369
- parentSelectorEntryRecordId: entryId,
1370
- childSelectorFamilyRecordId: childFamilyId,
1371
- relationKind: 'implementation-owns-selector',
1372
- }));
1373
- });
1374
- }
1375
- function javaImportedTypes(lines) {
1376
- const imports = new Map();
1377
- lines.forEach((line) => {
1378
- const imported = line.match(/^\s*import\s+(?:static\s+)?([A-Za-z_$][\w$]*(?:\.[A-Za-z_$][\w$]*)+)\s*;/)?.[1];
1379
- if (imported && !imported.endsWith('.*'))
1380
- imports.set(imported.split('.').at(-1), imported);
1381
- });
1382
- return imports;
1383
- }
1384
- function scanJavaSelectorCensus(root, paths, records) {
1385
- const beanAlternatives = [];
1386
- const commandFamilyIdsByClass = new Map();
1387
- for (const path of paths.filter((candidate) => candidate.endsWith('.java') && shouldScan(candidate))) {
1388
- let lines = null;
1389
- try {
1390
- lines = textLines(root, path);
1391
- }
1392
- catch {
1393
- lines = null;
1394
- }
1395
- if (!lines)
1396
- continue;
1397
- const source = lines.join('\n');
1398
- const imports = javaImportedTypes(lines);
1399
- const className = [...source.matchAll(/\b(?:class|interface|record)\s+([A-Za-z_$][\w$]*)/g)].at(-1)?.[1]
1400
- ?? basename(path, '.java');
1401
- const commandEntries = [];
1402
- for (let index = 0; index < lines.length; index += 1) {
1403
- if (!/@McpTool\s*\(/.test(lines[index]))
1404
- continue;
1405
- const annotation = [];
1406
- let cursor = index;
1407
- let depth = 0;
1408
- do {
1409
- annotation.push(lines[cursor]);
1410
- depth += parenthesisDelta(lines[cursor]);
1411
- cursor += 1;
1412
- } while (cursor < lines.length && depth > 0 && cursor - index < 20);
1413
- const annotationText = annotation.join('\n');
1414
- const selector = annotationText.match(/\bname\s*=\s*"([^"]+)"/)?.[1];
1415
- while (cursor < lines.length && (!lines[cursor].trim() || lines[cursor].trim().startsWith('@')))
1416
- cursor += 1;
1417
- const method = lines.slice(cursor, cursor + 6).join(' ').match(/\b(?:public|protected|private)?\s*(?:static\s+)?[A-Za-z_$][\w$]*(?:<[^;{}()]+>)?(?:\[\])?\s+([A-Za-z_$][\w$]*)\s*\(/)?.[1];
1418
- if (selector && method)
1419
- commandEntries.push({
1420
- selector,
1421
- implementation: `${path}#${className}.${method}`,
1422
- line: index + 1,
1423
- anchor: lines[index],
1424
- });
1425
- }
1426
- if (commandEntries.length > 0) {
1427
- const owner = `java-class:${className}`;
1428
- const identity = `annotation-dispatch:McpTool.name:${className}`;
1429
- const familyId = recordStableId('java-annotated-command-family', path, owner, identity);
1430
- commandFamilyIdsByClass.set(className, familyId);
1431
- addRecord(records, {
1432
- kind: 'java-annotated-command-family',
1433
- projections: ['registry_families'],
1434
- presumptiveInventory: true,
1435
- path,
1436
- line: commandEntries[0].line,
1437
- anchor: commandEntries[0].anchor,
1438
- owner,
1439
- identity,
1440
- tokens: ['McpTool.name', className],
1441
- });
1442
- commandEntries.forEach((entry) => addRecord(records, {
1443
- kind: 'java-annotated-command-entry',
1444
- projections: ['registry_entries'],
1445
- presumptiveInventory: true,
1446
- path,
1447
- line: entry.line,
1448
- anchor: entry.anchor,
1449
- owner,
1450
- identity: `${identity}:${entry.selector}:${entry.implementation}`,
1451
- tokens: [entry.selector, entry.implementation],
1452
- selectorKey: entry.selector,
1453
- implementationKey: entry.implementation,
1454
- selectorFamilyRecordId: familyId,
1455
- }));
1456
- }
1457
- for (const subtype of source.matchAll(/@JsonSubTypes\s*\(\s*\{([\s\S]*?)\}\s*\)\s*(?:public\s+)?(?:abstract\s+)?(?:class|interface)\s+([A-Za-z_$][\w$]*)/g)) {
1458
- const ownerType = subtype[2];
1459
- const entries = [...subtype[1].matchAll(/(?:JsonSubTypes\.)?Type\s*\(\s*value\s*=\s*([A-Za-z_$][\w$]*)\.class\s*,\s*name\s*=\s*([A-Za-z0-9_$."]+)/g)];
1460
- if (entries.length === 0)
1461
- continue;
1462
- const line = source.slice(0, subtype.index ?? 0).split('\n').length;
1463
- const owner = `java-type:${ownerType}`;
1464
- const identity = `json-subtypes:${ownerType}`;
1465
- const familyId = recordStableId('java-json-subtype-family', path, owner, identity);
1466
- addRecord(records, {
1467
- kind: 'java-json-subtype-family', projections: ['registry_families'], presumptiveInventory: true,
1468
- path, line, anchor: lines[line - 1] ?? '@JsonSubTypes', owner, identity, tokens: [ownerType],
1469
- });
1470
- entries.forEach((entry) => {
1471
- const implementation = imports.get(entry[1]) ?? entry[1];
1472
- const selector = entry[2].replace(/^"|"$/g, '');
1473
- addRecord(records, {
1474
- kind: 'java-json-subtype-entry', projections: ['registry_entries'], presumptiveInventory: true,
1475
- path, line, anchor: lines[line - 1] ?? '@JsonSubTypes', owner,
1476
- identity: `${identity}:${selector}:${implementation}`,
1477
- tokens: [selector, implementation], selectorKey: selector, implementationKey: implementation,
1478
- selectorFamilyRecordId: familyId,
1479
- });
1480
- });
1481
- }
1482
- const deserializers = [...source.matchAll(/\baddDeserializer\s*\(\s*([A-Za-z_$][\w$]*)\.class\s*,\s*new\s+([A-Za-z_$][\w$]*)\s*\(/g)];
1483
- if (deserializers.length > 0) {
1484
- const owner = `java-class:${className}`;
1485
- const identity = `jackson-deserializers:${className}`;
1486
- const familyId = recordStableId('java-deserializer-family', path, owner, identity);
1487
- const line = source.slice(0, deserializers[0].index ?? 0).split('\n').length;
1488
- addRecord(records, {
1489
- kind: 'java-deserializer-family', projections: ['registry_families'], presumptiveInventory: true,
1490
- path, line, anchor: lines[line - 1] ?? 'addDeserializer', owner, identity, tokens: [className],
1491
- });
1492
- deserializers.forEach((entry) => {
1493
- const entryLine = source.slice(0, entry.index ?? 0).split('\n').length;
1494
- addRecord(records, {
1495
- kind: 'java-deserializer-entry', projections: ['registry_entries'], presumptiveInventory: true,
1496
- path, line: entryLine, anchor: lines[entryLine - 1] ?? entry[0], owner,
1497
- identity: `${identity}:${entry[1]}:${entry[2]}`, tokens: [entry[1], entry[2]],
1498
- selectorKey: imports.get(entry[1]) ?? entry[1], implementationKey: imports.get(entry[2]) ?? entry[2],
1499
- selectorFamilyRecordId: familyId,
1500
- });
1501
- });
1502
- }
1503
- let pendingAnnotations = [];
1504
- let annotationBuffer = null;
1505
- lines.forEach((line, index) => {
1506
- const trimmed = line.trim();
1507
- if (annotationBuffer) {
1508
- annotationBuffer.text += ` ${trimmed}`;
1509
- annotationBuffer.depth += parenthesisDelta(line);
1510
- if (annotationBuffer.depth <= 0) {
1511
- pendingAnnotations.push({ line: annotationBuffer.line, text: annotationBuffer.text });
1512
- annotationBuffer = null;
1513
- }
1514
- return;
1515
- }
1516
- if (trimmed.startsWith('@')) {
1517
- const depth = parenthesisDelta(line);
1518
- if (depth > 0)
1519
- annotationBuffer = { line: index + 1, text: trimmed, depth };
1520
- else
1521
- pendingAnnotations.push({ line: index + 1, text: trimmed });
1522
- return;
1523
- }
1524
- if (!trimmed || trimmed.startsWith('//'))
1525
- return;
1526
- if (!pendingAnnotations.some((annotation) => /^@Bean\b/.test(annotation.text))) {
1527
- pendingAnnotations = [];
1528
- return;
1529
- }
1530
- const method = line.match(/^\s*(?:public|protected|private)?\s*(?:static\s+)?([A-Za-z_$][\w$]*(?:<[^;{}()]+>)?(?:\[\])?)\s+([A-Za-z_$][\w$]*)\s*\(/);
1531
- if (!method)
1532
- return;
1533
- const simpleContract = method[1].replace(/<.*$/, '');
1534
- const contract = imports.get(simpleContract) ?? simpleContract;
1535
- const conditions = pendingAnnotations
1536
- .filter((annotation) => !/^@Bean\b/.test(annotation.text))
1537
- .map((annotation) => annotation.text)
1538
- .sort();
1539
- beanAlternatives.push({
1540
- path,
1541
- line: index + 1,
1542
- anchor: line,
1543
- ownerClass: className,
1544
- method: method[2],
1545
- contract,
1546
- selector: conditions.length > 0
1547
- ? conditions.join('&')
1548
- : `unconditional:${path}#${className}.${method[2]}`,
1549
- implementation: `${path}#${className}.${method[2]}`,
1550
- });
1551
- pendingAnnotations = [];
1552
- });
1553
- }
1554
- const alternativesByContract = new Map();
1555
- beanAlternatives.forEach((alternative) => {
1556
- const group = alternativesByContract.get(alternative.contract) ?? [];
1557
- group.push(alternative);
1558
- alternativesByContract.set(alternative.contract, group);
1559
- });
1560
- alternativesByContract.forEach((alternatives, contract) => {
1561
- const exactAlternatives = [...new Map(alternatives.map((entry) => [entry.implementation, entry])).values()]
1562
- .sort((left, right) => left.implementation.localeCompare(right.implementation));
1563
- if (exactAlternatives.length < 2 || exactAlternatives.every((entry) => entry.selector.startsWith('unconditional:')))
1564
- return;
1565
- const anchor = exactAlternatives[0];
1566
- const owner = `spring-bean-contract:${contract}`;
1567
- const identity = `spring-bean-alternatives:${contract}`;
1568
- const familyId = recordStableId('java-spring-bean-alternative-family', anchor.path, owner, identity);
1569
- addRecord(records, {
1570
- kind: 'java-spring-bean-alternative-family', projections: ['registry_families'], presumptiveInventory: true,
1571
- path: anchor.path, line: anchor.line, anchor: anchor.anchor, owner, identity, tokens: [contract],
1572
- });
1573
- exactAlternatives.forEach((entry) => {
1574
- const entryIdentity = `${identity}:${entry.selector}:${entry.implementation}`;
1575
- const entryId = recordStableId('java-spring-bean-alternative-entry', entry.path, owner, entryIdentity);
1576
- addRecord(records, {
1577
- kind: 'java-spring-bean-alternative-entry', projections: ['registry_entries'], presumptiveInventory: true,
1578
- path: entry.path, line: entry.line, anchor: entry.anchor, owner, identity: entryIdentity,
1579
- tokens: [contract, entry.selector, entry.implementation], selectorKey: entry.selector,
1580
- implementationKey: entry.implementation, selectorFamilyRecordId: familyId,
1581
- });
1582
- const childFamilyId = commandFamilyIdsByClass.get(entry.contract.split('.').at(-1) ?? entry.contract);
1583
- if (!childFamilyId)
1584
- return;
1585
- addRecord(records, {
1586
- kind: 'java-selector-edge', projections: ['registry_edges'], presumptiveInventory: true,
1587
- path: entry.path, line: entry.line, anchor: entry.anchor, owner,
1588
- identity: `edge:${entryId}:${childFamilyId}`, tokens: [entryId, childFamilyId],
1589
- parentSelectorEntryRecordId: entryId, childSelectorFamilyRecordId: childFamilyId,
1590
- relationKind: 'implementation-owns-selector',
1591
- });
1592
- });
1593
- });
1594
- }
1595
- function scanJavaExactOperationMatrices(root, paths, records) {
1596
- const units = [];
1597
- for (const path of paths.filter((candidate) => candidate.endsWith('.java') && shouldScan(candidate))) {
1598
- let lines = null;
1599
- try {
1600
- lines = textLines(root, path);
1601
- }
1602
- catch {
1603
- lines = null;
1604
- }
1605
- if (!lines)
1606
- continue;
1607
- const source = lines.join('\n');
1608
- const className = source.match(/\b(?:class|record)\s+([A-Za-z_$][\w$]*)/)?.[1];
1609
- if (className)
1610
- units.push({ path, lines, source, className });
1611
- }
1612
- const unitsByClass = new Map(units.map((unit) => [unit.className, unit]));
1613
- units.forEach((declarationUnit) => {
1614
- for (const array of declarationUnit.source.matchAll(/\bString\s*\[\s*]\s*([A-Za-z_$][\w$]*)\s*=\s*\{([\s\S]*?)\}\s*;/g)) {
1615
- const symbol = array[1];
1616
- const values = [...array[2].matchAll(/"([^"]+)"/g)];
1617
- if (values.length < 2)
1618
- continue;
1619
- const accessor = `get${symbol[0].toUpperCase()}${symbol.slice(1)}`;
1620
- units.forEach((consumerUnit) => {
1621
- const accessorUse = new RegExp(`\\.${accessor}\\s*\\(\\s*\\)`, 'g');
1622
- for (const use of consumerUnit.source.matchAll(accessorUse)) {
1623
- const useOffset = use.index ?? 0;
1624
- const prefixStart = Math.max(0, useOffset - 1000);
1625
- const constructors = [...consumerUnit.source.slice(prefixStart, useOffset)
1626
- .matchAll(/new\s+([A-Za-z_$][\w$]*)\s*\(/g)];
1627
- const constructor = constructors.at(-1);
1628
- if (!constructor)
1629
- continue;
1630
- const implementation = unitsByClass.get(constructor[1]);
1631
- if (!implementation)
1632
- continue;
1633
- const httpMethods = [...new Set([...implementation.source.matchAll(/\.(get|post|put|patch|delete|options|head)\s*\(\s*\)/gi)]
1634
- .map((match) => match[1].toUpperCase()))];
1635
- if (httpMethods.length !== 1 || !/\.uri\s*\(/.test(implementation.source))
1636
- continue;
1637
- const method = httpMethods[0];
1638
- const arrayOffset = array.index ?? 0;
1639
- const declarationLine = declarationUnit.source.slice(0, arrayOffset).split('\n').length;
1640
- values.forEach((value, valueIndex) => {
1641
- const itemValue = value[1];
1642
- const itemOffset = arrayOffset + array[0].indexOf(array[2]) + (value.index ?? 0);
1643
- const itemLine = declarationUnit.source.slice(0, itemOffset).split('\n').length;
1644
- const configuredParts = itemValue.split(':');
1645
- const endpointId = configuredParts[0];
1646
- const endpointPath = configuredParts.length > 1 ? configuredParts.slice(1).join(':') : endpointId;
1647
- const itemIdentityKey = `${declarationUnit.path}#${declarationUnit.className}.${symbol}[${valueIndex}]`;
1648
- const operationIdentityKey = `${implementation.path}#${implementation.className}:${method}:matrix:${itemIdentityKey}`;
1649
- addRecord(records, {
1650
- kind: 'exact-operation-matrix-member',
1651
- projections: ['external_dependencies'],
1652
- presumptiveInventory: true,
1653
- path: declarationUnit.path,
1654
- line: itemLine,
1655
- anchor: declarationUnit.lines[itemLine - 1] ?? declarationUnit.lines[declarationLine - 1] ?? array[0],
1656
- owner: `${declarationUnit.className}.${symbol}`,
1657
- identity: operationIdentityKey,
1658
- tokens: [symbol, itemValue, endpointId, endpointPath, implementation.className, method],
1659
- operationToken: method,
1660
- addressToken: endpointPath,
1661
- operationIdentityKey,
1662
- matrixAxes: [{
1663
- symbol_identity_key: `${declarationUnit.path}#${declarationUnit.className}.${symbol}`,
1664
- item_identity_key: itemIdentityKey,
1665
- item_value: itemValue,
1666
- source: {
1667
- path: declarationUnit.path,
1668
- start_line: itemLine,
1669
- end_line: itemLine,
1670
- anchor: declarationUnit.lines[itemLine - 1] ?? itemValue,
1671
- },
1672
- }],
1673
- });
1674
- });
1675
- }
1676
- });
1677
- }
1678
- });
1679
- }
1680
- function enclosingTypeName(lines, index) {
1681
- for (let cursor = index; cursor >= 0; cursor -= 1) {
1682
- const type = lines[cursor].match(/\b(?:class|interface|record|enum)\s+([A-Za-z_$][\w$]*)/)?.[1];
1683
- if (type)
1684
- return type;
1685
- }
1686
- return basename('source');
1687
- }
1688
- function scanJavaAndBrowserOperationalIdentities(path, lines, records) {
1689
- const extension = extname(path.toLowerCase());
1690
- const javaLike = extension === '.java' || extension === '.kt' || extension === '.kts';
1691
- const browserLike = ['.js', '.jsx', '.mjs', '.cjs', '.ts', '.tsx', '.mts', '.cts', '.vue']
1692
- .includes(extension);
1693
- if (browserLike) {
1694
- lines.forEach((line, index) => {
1695
- for (const mount of line.matchAll(/\b([A-Za-z_$][\w$]*)\.mount\(\s*([^,)]+)/g)) {
1696
- const receiver = mount[1];
1697
- const address = mount[2].trim();
1698
- addRecord(records, {
1699
- kind: 'browser-mounted-application-role',
1700
- projections: ['runtime_roles'],
1701
- presumptiveInventory: true,
1702
- path,
1703
- line: index + 1,
1704
- anchor: line,
1705
- owner: `browser-application:${receiver}`,
1706
- identity: `mount:${receiver}:${address}`,
1707
- tokens: [receiver, address],
1708
- operationToken: 'mount',
1709
- addressToken: address,
1710
- operationIdentityKey: `${path}#${receiver}.mount:${address}`,
1711
- });
1712
- }
1713
- for (const storage of line.matchAll(/\b(?:window\.)?(localStorage|sessionStorage)\.(getItem|setItem|removeItem)\(\s*['"]([^'"]+)['"]/g))
1714
- addRecord(records, {
1715
- kind: 'browser-keyed-storage-state',
1716
- projections: ['stateful_resources'],
1717
- presumptiveInventory: true,
1718
- path,
1719
- line: index + 1,
1720
- anchor: line,
1721
- owner: `browser-storage:${storage[1]}`,
1722
- identity: `storage:${storage[1]}:${storage[3]}`,
1723
- tokens: [storage[1], storage[2], storage[3]],
1724
- addressToken: `${storage[1]}:${storage[3]}`,
1725
- operationToken: storage[2],
1726
- reproductionOperationToken: `${storage[1]}.${storage[2]}(${JSON.stringify(storage[3])})`,
1727
- });
1728
- for (const storage of line.matchAll(/\b(?:useStorage|useLocalStorage|useSessionStorage)\(\s*['"]([^'"]+)['"]/g))
1729
- addRecord(records, {
1730
- kind: 'browser-keyed-storage-state',
1731
- projections: ['stateful_resources'],
1732
- presumptiveInventory: true,
1733
- path,
1734
- line: index + 1,
1735
- anchor: line,
1736
- owner: 'browser-storage-composable',
1737
- identity: `storage-composable:${storage[1]}`,
1738
- tokens: [storage[0].split('(')[0], storage[1]],
1739
- addressToken: storage[1],
1740
- operationToken: storage[0].split('(')[0],
1741
- reproductionOperationToken: storage[0].split('(')[0],
1742
- });
1743
- const browserRequests = [...line.matchAll(/\b(axios\.(get|post|put|patch|delete|options|head))\(\s*([`'"])(.*?)\3/g)].map((request) => ({
1744
- callable: request[1],
1745
- operation: request[2].toUpperCase(),
1746
- address: request[4],
1747
- offset: request.index ?? 0,
1748
- anchor: line,
1749
- endLine: index + 1,
1750
- }));
1751
- for (const fetchStart of line.matchAll(/\bfetch\s*\(/g)) {
1752
- const offset = fetchStart.index ?? 0;
1753
- const callLines = [line.slice(offset)];
1754
- let depth = parenthesisDelta(callLines[0]);
1755
- let cursor = index;
1756
- while (depth > 0 && cursor + 1 < lines.length && cursor - index < 40) {
1757
- cursor += 1;
1758
- callLines.push(lines[cursor]);
1759
- depth += parenthesisDelta(lines[cursor]);
1760
- }
1761
- const callAnchor = callLines.join('\n');
1762
- const address = callAnchor.match(/\bfetch\s*\(\s*([`'"])(.*?)\1/s)?.[2];
1763
- if (!address)
1764
- continue;
1765
- const declaredMethod = directFetchMethod(callAnchor);
1766
- browserRequests.push({
1767
- callable: 'fetch',
1768
- operation: declaredMethod?.toUpperCase() ?? 'GET',
1769
- address,
1770
- offset,
1771
- anchor: callAnchor,
1772
- endLine: cursor + 1,
1773
- });
1774
- }
1775
- for (const request of browserRequests) {
1776
- addRecord(records, {
1777
- kind: 'browser-http-request-operation',
1778
- projections: ['external_dependencies'],
1779
- presumptiveInventory: true,
1780
- path,
1781
- line: index + 1,
1782
- endLine: request.endLine,
1783
- anchor: request.anchor,
1784
- owner: basename(path),
1785
- identity: `browser-http:${request.operation}:${request.address}@${index + 1}:${request.offset}`,
1786
- tokens: [request.callable, request.operation, request.address],
1787
- operationToken: request.operation,
1788
- addressToken: request.address,
1789
- operationIdentityKey: `${path}#browser-http:${request.operation}:${request.address}@${index + 1}:${request.offset}`,
1790
- });
1791
- }
1792
- });
1793
- const source = lines.join('\n');
1794
- for (const handle of source.matchAll(/(?:const|let|private|protected|public|readonly)\s+([A-Za-z_$][\w$]*)[^=;\n]*=\s*[^;\n]*\.subscribe\s*\(/g)) {
1795
- const field = handle[1];
1796
- if (!new RegExp(`\\b${field}\\.(?:unsubscribe|dispose|cancel)\\s*\\(`).test(source))
1797
- continue;
1798
- const line = source.slice(0, handle.index ?? 0).split('\n').length;
1799
- addRecord(records, {
1800
- kind: 'browser-supervised-subscription-role',
1801
- projections: ['runtime_roles'],
1802
- presumptiveInventory: true,
1803
- path,
1804
- line,
1805
- anchor: lines[line - 1] ?? field,
1806
- owner: basename(path),
1807
- identity: `subscription:${field}`,
1808
- tokens: [field, 'subscribe', 'unsubscribe'],
1809
- operationToken: 'subscribe/unsubscribe',
1810
- addressToken: field,
1811
- operationIdentityKey: `${path}#subscription:${field}`,
1812
- });
1813
- }
1814
- }
1815
- if (!javaLike)
1816
- return;
1817
- const source = lines.join('\n');
1818
- for (const bean of source.matchAll(/@Bean\s*\(\s*initMethod\s*=\s*"([^"]+)"\s*,\s*destroyMethod\s*=\s*"([^"]+)"\s*\)[\s\S]{0,500}?\b([A-Za-z_$][\w$<>?, ]*)\s+([A-Za-z_$][\w$]*)\s*\(/g)) {
1819
- const line = source.slice(0, bean.index ?? 0).split('\n').length;
1820
- addRecord(records, {
1821
- kind: 'spring-managed-lifecycle-role',
1822
- projections: ['runtime_roles'],
1823
- presumptiveInventory: true,
1824
- path,
1825
- line,
1826
- anchor: lines[line - 1] ?? '@Bean',
1827
- owner: `spring-bean:${bean[4]}`,
1828
- identity: `bean-lifecycle:${bean[4]}:${bean[1]}:${bean[2]}`,
1829
- tokens: [bean[4], bean[1], bean[2], bean[3].trim()],
1830
- operationToken: `${bean[1]}/${bean[2]}`,
1831
- addressToken: bean[4],
1832
- operationIdentityKey: `${path}#spring-bean:${bean[4]}:${bean[1]}:${bean[2]}`,
1833
- });
1834
- }
1835
- lines.forEach((line, index) => {
1836
- const staticResource = line.match(/\.addResourceHandler\s*\(([\s\S]*?)\)/);
1837
- if (staticResource) {
1838
- const route = [...staticResource[1].matchAll(/['"](\/[^'"]*)['"]/g)].at(-1)?.[1]
1839
- ?? staticResource[1].trim();
1840
- for (const method of ['GET', 'HEAD'])
1841
- addRecord(records, {
1842
- kind: 'spring-static-resource-inbound-operation',
1843
- projections: ['external_dependencies'],
1844
- presumptiveInventory: true,
1845
- path,
1846
- line: index + 1,
1847
- anchor: line,
1848
- owner: enclosingTypeName(lines, index),
1849
- identity: `static-resource:${method}:${route}@${index + 1}`,
1850
- tokens: [method, route],
1851
- operationToken: method,
1852
- addressToken: route,
1853
- operationIdentityKey: `${path}#static-resource:${method}:${route}@${index + 1}`,
1854
- });
1855
- }
1856
- const mapping = line.match(/@(Get|Post|Put|Patch|Delete|Request)Mapping\s*\((.*)\)/);
1857
- if (!mapping)
1858
- return;
1859
- const routes = [...mapping[2].matchAll(/['"](\/[^'"]*)['"]/g)].map((match) => match[1]);
1860
- const methods = mapping[1] === 'Request'
1861
- ? [...mapping[2].matchAll(/RequestMethod\.(GET|POST|PUT|PATCH|DELETE|OPTIONS|HEAD)/g)].map((match) => match[1])
1862
- : [mapping[1].toUpperCase()];
1863
- (methods.length > 0 ? methods : ['ANY']).forEach((method) => (routes.length > 0 ? routes : ['class-mapping']).forEach((route) => addRecord(records, {
1864
- kind: 'spring-controller-inbound-operation',
1865
- projections: ['external_dependencies'],
1866
- presumptiveInventory: true,
1867
- path,
1868
- line: index + 1,
1869
- anchor: line,
1870
- owner: enclosingTypeName(lines, index),
1871
- identity: `controller:${method}:${route}@${index + 1}`,
1872
- tokens: [method, route],
1873
- operationToken: method,
1874
- addressToken: route,
1875
- operationIdentityKey: `${path}#controller:${method}:${route}@${index + 1}`,
1876
- })));
1877
- });
1878
- const classBlocks = [...source.matchAll(/\bclass\s+([A-Za-z_$][\w$]*)/g)];
1879
- classBlocks.forEach((classMatch, classIndex) => {
1880
- const start = classMatch.index ?? 0;
1881
- const end = classBlocks[classIndex + 1]?.index ?? source.length;
1882
- const body = source.slice(start, end);
1883
- if (!/@PostConstruct\b/.test(body) || !/@PreDestroy\b/.test(body))
1884
- return;
1885
- const line = source.slice(0, start).split('\n').length;
1886
- addRecord(records, {
1887
- kind: 'java-annotated-lifecycle-role',
1888
- projections: ['runtime_roles'],
1889
- presumptiveInventory: true,
1890
- path,
1891
- line,
1892
- anchor: lines[line - 1] ?? classMatch[0],
1893
- owner: classMatch[1],
1894
- identity: `annotated-lifecycle:${classMatch[1]}`,
1895
- tokens: [classMatch[1], 'PostConstruct', 'PreDestroy'],
1896
- operationToken: 'PostConstruct/PreDestroy',
1897
- addressToken: classMatch[1],
1898
- operationIdentityKey: `${path}#class:${classMatch[1]}:PostConstruct/PreDestroy`,
1899
- });
1900
- });
1901
- for (const field of source.matchAll(/\b(?:Disposable|Subscription|ScheduledFuture(?:<[^>]+>)?)\s+([A-Za-z_$][\w$]*)\s*[;=]/g)) {
1902
- const fieldOffset = field.index ?? 0;
1903
- let enclosingClassIndex = -1;
1904
- for (let index = 0; index < classBlocks.length; index += 1) {
1905
- if ((classBlocks[index].index ?? 0) >= fieldOffset)
1906
- break;
1907
- enclosingClassIndex = index;
1908
- }
1909
- if (enclosingClassIndex >= 0) {
1910
- const classStart = classBlocks[enclosingClassIndex].index ?? 0;
1911
- const classEnd = classBlocks[enclosingClassIndex + 1]?.index ?? source.length;
1912
- const classBody = source.slice(classStart, classEnd);
1913
- if (/@PostConstruct\b/.test(classBody) && /@PreDestroy\b/.test(classBody))
1914
- continue;
1915
- }
1916
- const name = field[1];
1917
- const startPattern = new RegExp(`\\b(?:this\\.)?${name}\\s*=\\s*[\\s\\S]{0,300}?(?:subscribe|schedule(?:AtFixedRate|WithFixedDelay)?)\\s*\\(`);
1918
- const stopPattern = new RegExp(`\\b(?:this\\.)?${name}\\.(?:dispose|cancel|unsubscribe)\\s*\\(`);
1919
- if (!startPattern.test(source) || !stopPattern.test(source))
1920
- continue;
1921
- const line = source.slice(0, field.index ?? 0).split('\n').length;
1922
- const owner = enclosingTypeName(lines, line - 1);
1923
- addRecord(records, {
1924
- kind: 'java-supervised-worker-role',
1925
- projections: ['runtime_roles'],
1926
- presumptiveInventory: true,
1927
- path,
1928
- line,
1929
- anchor: lines[line - 1] ?? field[0],
1930
- owner,
1931
- identity: `supervised-worker:${owner}:${name}`,
1932
- tokens: [owner, name, field[0].trim()],
1933
- operationToken: 'start/stop-handle',
1934
- addressToken: name,
1935
- operationIdentityKey: `${path}#${owner}:worker:${name}`,
1936
- });
1937
- }
1938
- }
1939
- const EFFECT_PATTERNS = [
1940
- { kind: 'filesystem-read-operation', projections: ['external_dependencies'], regex: /(?:\b(?:readFile(?:Sync)?|read_text|read_bytes|File\.read|fs\.readFile)|(?<!\.)\bopen)\s*\(/g },
1941
- { kind: 'filesystem-write-operation', projections: ['stateful_resources', 'external_dependencies'], regex: /\b(?:writeFile(?:Sync)?|appendFile(?:Sync)?|write_text|write_bytes|touch|mkdir|makedirs|unlink|rmdir|rmtree|File\.write|fs\.writeFile)\s*\(/g },
1942
- { kind: 'filesystem-metadata-operation', projections: ['external_dependencies'], regex: /\b(?:os\.(?:stat|lstat|listdir|scandir|walk|readlink|getcwd|chdir|chmod|rename|replace)|glob\.(?:glob|iglob)|shutil\.(?:copy|copy2|copyfile|copytree|move)|[A-Za-z0-9_.]+\.(?:exists|is_file|is_dir|stat|glob|rglob|iterdir|resolve|readlink|chmod|rename|replace|symlink_to|hardlink_to))\s*\(/g },
1943
- { kind: 'subprocess-operation', projections: ['external_dependencies'], regex: /\b(?:subprocess\.(?:run|Popen|call|check_call|check_output)|create_subprocess_(?:exec|shell)|child_process\.(?:spawn|exec|execFile|fork)|Command::new|ProcessBuilder|os\.(?:system|exec\w*))\s*\(/g },
1944
- { kind: 'command-wrapper-operation', projections: ['external_dependencies'], regex: /\b(?:run_command|run_cmd|execute_command|exec_command|invoke_command|shell_command)\s*\(\s*['"][^'"]+['"]/g },
1945
- { kind: 'host-execution-operation', projections: ['external_dependencies'], regex: /\b(?:runpy\.(?:run_module|run_path)|importlib\.(?:import_module|metadata\.(?:entry_points|distributions|version))|pkg_resources\.(?:iter_entry_points|get_distribution)|compile|exec)\s*\(/g },
1946
- { kind: 'host-installation-operation', projections: ['external_dependencies'], regex: /\b(?:entry_points|distributions|distribution|metadata|version)\s*\(|\.load\s*\(\s*\)/g },
1947
- { kind: 'host-interpreter-operation', projections: ['stateful_resources', 'external_dependencies'], regex: /\b(?:sys\.(?:meta_path|path_hooks|path_importer_cache|modules)|importlib\.(?:invalidate_caches|reload)|inspect\.(?:getsource|getattr_static|getmodule))\b/g },
1948
- { kind: 'socket-operation', projections: ['external_dependencies'], regex: /(?:\bsocket\.create_connection|\.(?:bind|listen|accept|connect|sendto|recvfrom))\s*\(/g },
1949
- { kind: 'host-signal-operation', projections: ['external_dependencies'], regex: /\b(?:signal\.signal|os\.kill|process\.(?:on|once)|[A-Za-z0-9_.]+\.add_signal_handler)\s*\(/g },
1950
- { kind: 'browser-host-operation', projections: ['external_dependencies'], regex: /\b(?:webbrowser\.open|window\.(?:open|addEventListener|removeEventListener)|document\.(?:createElement|createTextNode|getElementById|querySelector|querySelectorAll)|navigator\.[A-Za-z0-9_]+|Browser\.open|openExternal)\s*\(/g },
1951
- { kind: 'browser-model-operation', projections: ['external_dependencies'], regex: /\b(?:model|view)\.(?:addEventListener|addSignalListener|change|data|finalize|get|height|off|on|removeEventListener|removeSignalListener|resize|run|runAsync|save_changes|send|set|signal|toCanvas|toImageURL|toSVG|width)\s*\(/g },
1952
- { kind: 'browser-module-loader-operation', projections: ['external_dependencies'], regex: /\b(?:import|require|requirejs|define)(?:\.config)?\s*\(/g },
1953
- { kind: 'http-operation', projections: ['external_dependencies'], regex: /\b(?:fetch|requests\.(?:get|post|put|patch|delete|request)|axios\.(?:get|post|put|patch|delete|request)|httpx\.(?:get|post|put|patch|delete|request))\s*\(/g },
1954
- { kind: 'http-wrapper-operation', projections: ['external_dependencies'], regex: /\b(?:self\.)?_(?:get|post|put|patch|delete|request)(?:_raw)?\s*\(\s*(?:f|r|fr|rf)?['"]/g },
1955
- { kind: 'client-boundary-operation', projections: ['external_dependencies'], regex: /\b(?:[A-Za-z_][A-Za-z0-9_]*\.)*[A-Za-z0-9_]*(?:api|channel|client|connection|exporter|http|provider|session|transport)[A-Za-z0-9_]*\.(?:get|post|put|patch|delete|request|send|receive|respond|stream|query|execute|create|update|remove|list|load|export|publish|consume|ack|invoke|call)\s*\(/gi },
1956
- { kind: 'sdk-boundary-operation', projections: ['external_dependencies'], regex: /\b[A-Za-z_][A-Za-z0-9_.]*\.(?:add_event|add_link|attach|detach|emit|end|export|extract|force_flush|inject|invoke|observe|publish|record|receive_message|send_message|set_attribute|set_attributes|set_status|shutdown|start_span|write)\s*\(/g },
1957
- { kind: 'listener-operation', projections: ['external_dependencies'], regex: /\b(?:listen|serve|Server|Route|router\.(?:get|post|put|patch|delete)|app\.(?:get|post|put|patch|delete)|add_url_rule)\s*\(/g },
1958
- { kind: 'host-metadata-operation', projections: ['external_dependencies'], regex: /\b(?:socket\.gethostname|platform\.(?:machine|system|platform|node)|os\.(?:getpid|getppid|getuid|getgid|uname)|psutil\.(?:Process|cpu_percent|cpu_count|virtual_memory)|process\.(?:cpuUsage|memoryUsage))\s*\(/g },
1959
- { kind: 'installed-callback-state', projections: ['stateful_resources', 'external_dependencies'], regex: /\b(?:atexit\.register|os\.register_at_fork|signal\.signal|set_global_[A-Za-z0-9_]+|setattr|monkeypatch|patch)(?:\s*\(|\s*$)/g },
1960
- { kind: 'installed-registration-state', projections: ['stateful_resources'], regex: /\.(?:add_handler|add_hook|add_middleware|add_instrumentation|instrument)\s*\(/g },
1961
- { kind: 'in-memory-state', projections: ['stateful_resources'], regex: /\b(?:Queue|SimpleQueue|deque|WeakSet|WeakKeyDictionary|WeakValueDictionary|LRUCache|TTLCache)\s*\(/g },
1962
- { kind: 'runtime-task-candidate', projections: ['runtime_roles'], regex: /\b(?:asyncio\.create_task|create_task|ensure_future|Thread|Process|ThreadPoolExecutor|ProcessPoolExecutor)\s*\(/g, presumptive: false },
1963
- ];
1964
- function scanEffects(path, lines, owners, executable, records) {
1965
- lines.forEach((line, index) => {
1966
- if (!executable[index])
1967
- return;
1968
- const owner = owners[index] ?? basename(path);
1969
- for (const pattern of EFFECT_PATTERNS) {
1970
- pattern.regex.lastIndex = 0;
1971
- let match;
1972
- while ((match = pattern.regex.exec(line)) !== null) {
1973
- const token = match[0].replace(/\s*\($/, '').trim();
1974
- addRecord(records, {
1975
- kind: pattern.kind,
1976
- projections: pattern.projections,
1977
- presumptiveInventory: pattern.presumptive ?? true,
1978
- path,
1979
- line: index + 1,
1980
- anchor: line,
1981
- owner,
1982
- identity: `${token}@${index + 1}:${match.index}`,
1983
- tokens: [token],
1984
- operationToken: token,
1985
- });
1986
- if (match[0].length === 0)
1987
- pattern.regex.lastIndex += 1;
1988
- }
1989
- }
1990
- const envMatches = [
1991
- ...line.matchAll(/os\.getenv\(\s*['"]([^'"]+)['"]/g),
1992
- ...line.matchAll(/os\.environ\.(get|pop|setdefault)\(\s*['"]([^'"]+)['"]/g),
1993
- ...line.matchAll(/os\.environ\[\s*['"]([^'"]+)['"]\s*\]/g),
1994
- ...line.matchAll(/process\.env\.([A-Za-z_][A-Za-z0-9_]*)/g),
1995
- ...line.matchAll(/process\.env\[\s*['"]([^'"]+)['"]\s*\]/g),
1996
- ].sort((left, right) => (left.index ?? 0) - (right.index ?? 0));
1997
- for (const envMatch of envMatches) {
1998
- const method = envMatch[1] && ['get', 'pop', 'setdefault'].includes(envMatch[1]) ? envMatch[1] : null;
1999
- const key = method ? envMatch[2] : envMatch[1];
2000
- const suffix = line.slice((envMatch.index ?? 0) + envMatch[0].length);
2001
- const written = method === 'pop' || method === 'setdefault' || /^\s*=/.test(suffix);
2002
- addRecord(records, {
2003
- kind: written ? 'environment-write-operation' : 'environment-read-operation',
2004
- projections: written ? ['stateful_resources', 'external_dependencies'] : ['external_dependencies'],
2005
- presumptiveInventory: true,
2006
- path, line: index + 1, anchor: line, owner,
2007
- identity: `environment:${written ? 'write' : 'read'}:${key}@${index + 1}`,
2008
- tokens: [key], operationToken: written ? 'environment-write' : 'environment-read', addressToken: key,
2009
- });
2010
- }
2011
- const dynamicEnvMatches = [
2012
- ...line.matchAll(/os\.getenv\(\s*([^,)]+)(?:,|\))/g),
2013
- ...line.matchAll(/os\.environ\.(get|pop|setdefault)\(\s*([^,)]+)(?:,|\))/g),
2014
- ...line.matchAll(/os\.environ\[\s*([^\]]+)\s*\]/g),
2015
- ].sort((left, right) => (left.index ?? 0) - (right.index ?? 0));
2016
- for (const envMatch of dynamicEnvMatches) {
2017
- const method = envMatch[1] && ['get', 'pop', 'setdefault'].includes(envMatch[1])
2018
- ? envMatch[1]
2019
- : null;
2020
- const expression = (method ? envMatch[2] : envMatch[1])?.trim();
2021
- if (!expression || /^['"][^'"]+['"]$/.test(expression))
2022
- continue;
2023
- const suffix = line.slice((envMatch.index ?? 0) + envMatch[0].length);
2024
- const written = method === 'pop' || method === 'setdefault' || /^\s*=/.test(suffix);
2025
- addRecord(records, {
2026
- kind: written ? 'environment-write-operation' : 'environment-read-operation',
2027
- projections: written ? ['stateful_resources', 'external_dependencies'] : ['external_dependencies'],
2028
- presumptiveInventory: true,
2029
- path, line: index + 1, anchor: line, owner,
2030
- identity: `environment:${written ? 'write' : 'read'}:${expression}@${index + 1}`,
2031
- tokens: [expression], operationToken: written ? 'environment-write' : 'environment-read',
2032
- addressToken: expression,
2033
- });
2034
- }
2035
- const synchronizedTrait = line.match(/^\s*([A-Za-z_][A-Za-z0-9_]*)\s*(?::[^=]+)?=.+\.tag\([^)]*\bsync\s*=\s*(?:True|true)\b/);
2036
- if (synchronizedTrait)
2037
- addRecord(records, {
2038
- kind: 'synchronized-trait-state',
2039
- projections: ['stateful_resources'],
2040
- presumptiveInventory: true,
2041
- path,
2042
- line: index + 1,
2043
- anchor: line,
2044
- owner,
2045
- identity: `synchronized-trait:${synchronizedTrait[1]}@${index + 1}`,
2046
- tokens: [synchronizedTrait[1], 'sync'],
2047
- addressToken: synchronizedTrait[1],
2048
- operationToken: 'sync-trait',
2049
- });
2050
- for (const resourceAssignment of line.matchAll(/\b([A-Za-z_][A-Za-z0-9_.]*)\.(src|href)\s*=\s*([^;]+)/g))
2051
- addRecord(records, {
2052
- kind: 'browser-resource-load-operation',
2053
- projections: ['external_dependencies'],
2054
- presumptiveInventory: true,
2055
- path,
2056
- line: index + 1,
2057
- anchor: line,
2058
- owner,
2059
- identity: `browser-resource:${resourceAssignment[1]}.${resourceAssignment[2]}:${resourceAssignment[3].trim()}@${index + 1}`,
2060
- tokens: [resourceAssignment[1], resourceAssignment[2], resourceAssignment[3].trim()],
2061
- operationToken: `${resourceAssignment[1]}.${resourceAssignment[2]}=`,
2062
- addressToken: resourceAssignment[3].trim(),
2063
- });
2064
- const openWrite = line.match(/\bopen\s*\([^,\n]+,\s*(?:mode\s*=\s*)?['"]([wax+][^'"]*)['"]/) ??
2065
- line.match(/\b[A-Za-z_][A-Za-z0-9_.]*\.open\s*\(\s*['"]([wax+][^'"]*)['"]/);
2066
- if (openWrite)
2067
- addRecord(records, {
2068
- kind: 'filesystem-write-operation', projections: ['stateful_resources', 'external_dependencies'], presumptiveInventory: true,
2069
- path, line: index + 1, anchor: line, owner,
2070
- identity: `open-write:${openWrite[1]}@${index + 1}`, tokens: [openWrite[1]], operationToken: 'open-write',
2071
- });
2072
- for (const mutation of line.matchAll(/\b((?:self\.)?[A-Za-z_][A-Za-z0-9_]*(?:\.[A-Za-z_][A-Za-z0-9_]*)*)\.(append|extend|add|set|update|pop|remove|discard|clear|put|put_nowait|setdefault)\s*\(/g)) {
2073
- const target = mutation[1];
2074
- if (!target.startsWith('self.') &&
2075
- !/^[A-Z][A-Z0-9_]*$/.test(target) &&
2076
- !/(?:cache|config|credential|hook|open_span|queue|registry|session|state|store|token|variable)/i.test(target))
2077
- continue;
2078
- addRecord(records, {
2079
- kind: 'in-memory-state-mutation', projections: ['stateful_resources'], presumptiveInventory: true,
2080
- path, line: index + 1, anchor: line, owner,
2081
- identity: `memory-mutation:${target}.${mutation[2]}@${index + 1}`,
2082
- tokens: [target, mutation[2]], operationToken: mutation[2], addressToken: target,
2083
- });
2084
- }
2085
- const indexedMutation = line.match(/^\s*((?:self\.)?[A-Za-z_][A-Za-z0-9_]*(?:\.[A-Za-z_][A-Za-z0-9_]*)*)\s*\[[^\]]+\]\s*=/);
2086
- if (indexedMutation &&
2087
- (indexedMutation[1].startsWith('self.') ||
2088
- /^[A-Z][A-Z0-9_]*$/.test(indexedMutation[1]) ||
2089
- /(?:cache|config|credential|queue|registry|session|state|store|token|variable)/i.test(indexedMutation[1])))
2090
- addRecord(records, {
2091
- kind: 'in-memory-state-mutation', projections: ['stateful_resources'], presumptiveInventory: true,
2092
- path, line: index + 1, anchor: line, owner,
2093
- identity: `memory-index-write:${indexedMutation[1]}@${index + 1}`,
2094
- tokens: [indexedMutation[1]], operationToken: 'indexed-write', addressToken: indexedMutation[1],
2095
- });
2096
- for (const remoteMutation of line.matchAll(/\b((?:[A-Za-z_][A-Za-z0-9_]*\.)+[A-Za-z_][A-Za-z0-9_]*)\.(post|put|patch|delete|create|update|remove|publish)\s*\(/gi)) {
2097
- addRecord(records, {
2098
- kind: 'remote-state-mutation-operation', projections: ['stateful_resources', 'external_dependencies'], presumptiveInventory: true,
2099
- path, line: index + 1, anchor: line, owner,
2100
- identity: `remote-mutation:${remoteMutation[1]}.${remoteMutation[2]}@${index + 1}`,
2101
- tokens: [remoteMutation[1], remoteMutation[2]], operationToken: remoteMutation[2].toLowerCase(),
2102
- });
2103
- }
2104
- const stateAddress = line.match(/^\s*((?:self\.)?[A-Za-z_][A-Za-z0-9_]*(?:PATH|FILE|DIR|HOME|CACHE|CONFIG|STORE|TOKEN|CREDENTIAL|DATABASE)[A-Za-z0-9_]*)\s*(?::[^=]+)?=\s*(.+)$/i);
2105
- if (stateAddress && /(?:\bPath\s*\(|\/\s*['"]|['"][^'"]*(?:\/|\.[A-Za-z0-9]{1,8})['"])/.test(stateAddress[2])) {
2106
- addRecord(records, {
2107
- kind: 'state-address-declaration', projections: ['stateful_resources'], presumptiveInventory: true,
2108
- path, line: index + 1, anchor: line, owner,
2109
- identity: `state-address:${stateAddress[1]}@${index + 1}`,
2110
- tokens: [stateAddress[1], stateAddress[2].trim()], addressToken: stateAddress[1],
2111
- });
2112
- }
2113
- const dataclassState = line.match(/^\s*([A-Za-z_][A-Za-z0-9_]*)\s*:\s*[^=]+?=\s*field\(\s*default_factory\s*=\s*([A-Za-z_][A-Za-z0-9_.]*)/);
2114
- if (dataclassState)
2115
- addRecord(records, {
2116
- kind: 'in-memory-state', projections: ['stateful_resources'], presumptiveInventory: true,
2117
- path, line: index + 1, anchor: line, owner,
2118
- identity: `dataclass-state:${dataclassState[1]}:${dataclassState[2]}@${index + 1}`,
2119
- tokens: [dataclassState[1], dataclassState[2]], addressToken: dataclassState[1],
2120
- });
2121
- const mutableAssignment = line.match(/^\s*(?:self\.)?([A-Za-z_][A-Za-z0-9_]*)\s*(?::[^=]+)?=\s*(\{\}|\[\]|set\(\)|dict\(\)|list\(\))/);
2122
- if (mutableAssignment)
2123
- addRecord(records, {
2124
- kind: 'in-memory-state', projections: ['stateful_resources'], presumptiveInventory: false,
2125
- path, line: index + 1, anchor: line, owner,
2126
- identity: `mutable:${mutableAssignment[1]}@${index + 1}`, tokens: [mutableAssignment[1], mutableAssignment[2]],
2127
- addressToken: mutableAssignment[1],
2128
- });
2129
- const wrapperAssignment = line.match(/^\s*([A-Za-z_][A-Za-z0-9_.]*)\s*=\s*([A-Za-z_][A-Za-z0-9_.]*)\s*(?:#.*)?$/);
2130
- const wrapperRoot = wrapperAssignment?.[1].split('.')[0] ?? '';
2131
- const wrapperTarget = wrapperAssignment?.[2] ?? '';
2132
- const looksLikeInstalledWrapper = Boolean(wrapperAssignment
2133
- && wrapperAssignment[1].includes('.')
2134
- && (/^[A-Z]/.test(wrapperRoot)
2135
- || /^(?:builtins|os|sys|asyncio|concurrent|threading|socket|signal)\./.test(wrapperAssignment[1])
2136
- || /(?:wrap|patch|handler|hook|instrument|send|receive|respond)/i.test(wrapperTarget)));
2137
- if (wrapperAssignment && looksLikeInstalledWrapper)
2138
- addRecord(records, {
2139
- kind: 'installed-wrapper-state', projections: ['stateful_resources', 'external_dependencies'], presumptiveInventory: true,
2140
- path, line: index + 1, anchor: line, owner,
2141
- identity: `patch:${wrapperAssignment[1]}:${wrapperAssignment[2]}@${index + 1}`,
2142
- tokens: [wrapperAssignment[1], wrapperAssignment[2]], operationToken: wrapperAssignment[1], addressToken: wrapperAssignment[1],
2143
- });
2144
- const routeMethods = [...line.matchAll(/['"](GET|POST|PUT|PATCH|DELETE|OPTIONS|HEAD)['"]/g)].map((match) => match[1]);
2145
- if (/\b(?:Route|router\.|app\.|add_url_rule)/.test(line) && routeMethods.length > 0) {
2146
- const route = line.match(/(?:f|r|fr|rf)?['"](\/[^'"]*)['"]/)?.[1];
2147
- routeMethods.forEach((method) => addRecord(records, {
2148
- kind: 'required-inbound-operation', projections: ['external_dependencies'], presumptiveInventory: true,
2149
- path, line: index + 1, anchor: line, owner,
2150
- identity: `inbound:${method}:${route ?? 'route'}@${index + 1}`,
2151
- tokens: [method, route ?? 'route'], operationToken: method, addressToken: route,
2152
- }));
2153
- }
2154
- });
2155
- }
2156
- function parenthesisDelta(value) {
2157
- let delta = 0;
2158
- let quote = null;
2159
- let escaped = false;
2160
- for (const char of value) {
2161
- if (escaped) {
2162
- escaped = false;
2163
- continue;
2164
- }
2165
- if (char === '\\') {
2166
- escaped = true;
2167
- continue;
2168
- }
2169
- if (quote) {
2170
- if (char === quote)
2171
- quote = null;
2172
- continue;
2173
- }
2174
- if (char === "'" || char === '"') {
2175
- quote = char;
2176
- continue;
2177
- }
2178
- if (char === '(')
2179
- delta += 1;
2180
- if (char === ')')
2181
- delta -= 1;
2182
- }
2183
- return delta;
2184
- }
2185
- function splitTopLevelJavaScript(value) {
2186
- const segments = [];
2187
- let start = 0;
2188
- let parentheses = 0;
2189
- let brackets = 0;
2190
- let braces = 0;
2191
- let quote = null;
2192
- let escaped = false;
2193
- let lineComment = false;
2194
- let blockComment = false;
2195
- for (let index = 0; index < value.length; index += 1) {
2196
- const char = value[index];
2197
- const next = value[index + 1];
2198
- if (lineComment) {
2199
- if (char === '\n')
2200
- lineComment = false;
2201
- continue;
2202
- }
2203
- if (blockComment) {
2204
- if (char === '*' && next === '/') {
2205
- blockComment = false;
2206
- index += 1;
2207
- }
2208
- continue;
2209
- }
2210
- if (escaped) {
2211
- escaped = false;
2212
- continue;
2213
- }
2214
- if (quote) {
2215
- if (char === '\\')
2216
- escaped = true;
2217
- else if (char === quote)
2218
- quote = null;
2219
- continue;
2220
- }
2221
- if (char === '/' && next === '/') {
2222
- lineComment = true;
2223
- index += 1;
2224
- continue;
2225
- }
2226
- if (char === '/' && next === '*') {
2227
- blockComment = true;
2228
- index += 1;
2229
- continue;
2230
- }
2231
- if (char === "'" || char === '"' || char === '`') {
2232
- quote = char;
2233
- continue;
2234
- }
2235
- if (char === '(')
2236
- parentheses += 1;
2237
- else if (char === ')')
2238
- parentheses -= 1;
2239
- else if (char === '[')
2240
- brackets += 1;
2241
- else if (char === ']')
2242
- brackets -= 1;
2243
- else if (char === '{')
2244
- braces += 1;
2245
- else if (char === '}')
2246
- braces -= 1;
2247
- else if (char === ',' && parentheses === 0 && brackets === 0 && braces === 0) {
2248
- segments.push(value.slice(start, index));
2249
- start = index + 1;
2250
- }
2251
- }
2252
- segments.push(value.slice(start));
2253
- return segments;
2254
- }
2255
- function javascriptCallArguments(call) {
2256
- const open = call.indexOf('(');
2257
- if (open < 0)
2258
- return [];
2259
- let depth = 0;
2260
- let quote = null;
2261
- let escaped = false;
2262
- let lineComment = false;
2263
- let blockComment = false;
2264
- for (let index = open + 1; index < call.length; index += 1) {
2265
- const char = call[index];
2266
- const next = call[index + 1];
2267
- if (lineComment) {
2268
- if (char === '\n')
2269
- lineComment = false;
2270
- continue;
2271
- }
2272
- if (blockComment) {
2273
- if (char === '*' && next === '/') {
2274
- blockComment = false;
2275
- index += 1;
2276
- }
2277
- continue;
2278
- }
2279
- if (escaped) {
2280
- escaped = false;
2281
- continue;
2282
- }
2283
- if (quote) {
2284
- if (char === '\\')
2285
- escaped = true;
2286
- else if (char === quote)
2287
- quote = null;
2288
- continue;
2289
- }
2290
- if (char === '/' && next === '/') {
2291
- lineComment = true;
2292
- index += 1;
2293
- continue;
2294
- }
2295
- if (char === '/' && next === '*') {
2296
- blockComment = true;
2297
- index += 1;
2298
- continue;
2299
- }
2300
- if (char === "'" || char === '"' || char === '`') {
2301
- quote = char;
2302
- continue;
2303
- }
2304
- if (char === '(')
2305
- depth += 1;
2306
- else if (char === ')' && depth > 0)
2307
- depth -= 1;
2308
- else if (char === ')')
2309
- return splitTopLevelJavaScript(call.slice(open + 1, index));
2310
- }
2311
- return [];
2312
- }
2313
- function directFetchMethod(call) {
2314
- const init = javascriptCallArguments(call)[1]?.trim();
2315
- if (!init?.startsWith('{'))
2316
- return undefined;
2317
- let close = -1;
2318
- let depth = 0;
2319
- let quote = null;
2320
- let escaped = false;
2321
- for (let index = 0; index < init.length; index += 1) {
2322
- const char = init[index];
2323
- if (escaped) {
2324
- escaped = false;
2325
- continue;
2326
- }
2327
- if (quote) {
2328
- if (char === '\\')
2329
- escaped = true;
2330
- else if (char === quote)
2331
- quote = null;
2332
- continue;
2333
- }
2334
- if (char === "'" || char === '"' || char === '`') {
2335
- quote = char;
2336
- continue;
2337
- }
2338
- if (char === '{')
2339
- depth += 1;
2340
- else if (char === '}') {
2341
- depth -= 1;
2342
- if (depth === 0) {
2343
- close = index;
2344
- break;
2345
- }
2346
- }
2347
- }
2348
- if (close < 0)
2349
- return undefined;
2350
- let method;
2351
- for (const property of splitTopLevelJavaScript(init.slice(1, close))) {
2352
- const direct = property.match(/^\s*(?:method|['"]method['"])\s*:\s*([\s\S]*)$/i);
2353
- if (!direct)
2354
- continue;
2355
- method = direct[1].match(/^\s*(['"])(GET|POST|PUT|PATCH|DELETE|OPTIONS|HEAD)\1(?:\s|$)/i)?.[2];
2356
- }
2357
- return method;
2358
- }
2359
- function scanExternalCallBlocks(path, lines, owners, executable, records) {
2360
- for (let index = 0; index < lines.length; index += 1) {
2361
- if (!executable[index])
2362
- continue;
2363
- const start = lines[index];
2364
- const call = start.match(/\b([A-Za-z_][A-Za-z0-9_.]*)\s*\(/);
2365
- if (!call)
2366
- continue;
2367
- let depth = parenthesisDelta(start.slice(call.index ?? 0));
2368
- const block = [start];
2369
- let end = index;
2370
- while (depth > 0 && end + 1 < lines.length && end - index < 40) {
2371
- end += 1;
2372
- block.push(lines[end]);
2373
- depth += parenthesisDelta(lines[end]);
2374
- }
2375
- const blockText = block.join('\n');
2376
- const callable = call[1];
2377
- const shortCallable = callable.split('.').at(-1) ?? callable;
2378
- const owner = owners[index] ?? basename(path);
2379
- const callLiterals = literalTokens(blockText);
2380
- if (shortCallable === 'register' && callLiterals.length > 0) {
2381
- const selector = callLiterals[0];
2382
- const argumentText = blockText.slice(blockText.indexOf('(') + 1, blockText.lastIndexOf(')'));
2383
- const implementation = argumentText.split(',').slice(1).join(',').trim() || 'registered-callback';
2384
- addRecord(records, {
2385
- kind: 'installed-registration-state',
2386
- projections: ['stateful_resources'],
2387
- presumptiveInventory: true,
2388
- path,
2389
- line: index + 1,
2390
- anchor: start,
2391
- owner,
2392
- identity: `installed-registration:${callable}:${selector}:${implementation}@${index + 1}`,
2393
- tokens: [callable, selector, implementation],
2394
- selectorKey: selector,
2395
- implementationKey: implementation,
2396
- operationToken: callable,
2397
- addressToken: selector,
2398
- });
2399
- }
2400
- if (/^(?:addEventListener|addListener|observe|on|register_magic_function|subscribe|watch)$/.test(shortCallable)) {
2401
- const selector = callLiterals.join('|') || 'all-events';
2402
- addRecord(records, {
2403
- kind: 'callback-registration-operation',
2404
- projections: path.endsWith('.py')
2405
- ? ['stateful_resources', 'external_dependencies']
2406
- : ['external_dependencies'],
2407
- presumptiveInventory: true,
2408
- path,
2409
- line: index + 1,
2410
- anchor: start,
2411
- owner,
2412
- identity: `callback-registration:${callable}:${selector}@${index + 1}`,
2413
- tokens: [callable, ...callLiterals],
2414
- selectorKey: selector,
2415
- implementationKey: `callback@${path}:${index + 1}`,
2416
- operationToken: callable,
2417
- addressToken: selector,
2418
- });
2419
- }
2420
- const constructorBoundary = /(?:Api|Channel|Client|Connection|Consumer|Exporter|Producer|Provider|Publisher|Reader|Session|Stub|Transport|Writer)$/i.test(shortCallable) &&
2421
- /(?:endpoint|base_url|url|host|address|dsn|channel|connection|session|transport|headers|credentials)\s*=/i.test(blockText);
2422
- if (constructorBoundary) {
2423
- addRecord(records, {
2424
- kind: 'external-constructor-operation', projections: ['external_dependencies'], presumptiveInventory: true,
2425
- path, line: index + 1, anchor: start, owner,
2426
- identity: `external-constructor:${callable}@${index + 1}`,
2427
- tokens: [callable, ...literalTokens(blockText).slice(0, 8)],
2428
- operationToken: callable,
2429
- });
2430
- }
2431
- if (/(?:^|\.)(?:run|Popen|call|check_call|check_output|spawn|exec|execFile)$/.test(callable) &&
2432
- /['"](?:add|configure|create|delete|disable|enable|install|patch|register|remove|set|update)['"]/.test(blockText)) {
2433
- addRecord(records, {
2434
- kind: 'installed-command-state', projections: ['stateful_resources', 'external_dependencies'], presumptiveInventory: true,
2435
- path, line: index + 1, anchor: start, owner,
2436
- identity: `installed-command:${callable}:${literalTokens(blockText).join(':')}@${index + 1}`,
2437
- tokens: [callable, ...literalTokens(blockText).slice(0, 10)], operationToken: callable,
2438
- });
2439
- }
2440
- if (/^(?:self\.)?_(?:get|post|put|patch|delete|request)(?:_raw)?$/i.test(callable)) {
2441
- const address = blockText.match(/(?:f|r|fr|rf)?['"]([^'"]*(?:\/|https?:)[^'"]*)['"]/)?.[1];
2442
- if (address)
2443
- addRecord(records, {
2444
- kind: 'http-wrapper-operation',
2445
- projections: /_(?:post|put|patch|delete)(?:_raw)?$/i.test(callable)
2446
- ? ['stateful_resources', 'external_dependencies']
2447
- : ['external_dependencies'],
2448
- presumptiveInventory: true,
2449
- path, line: index + 1, anchor: start, owner,
2450
- identity: `http-wrapper:${callable}:${address}@${index + 1}`,
2451
- tokens: [callable, address], operationToken: callable, addressToken: address,
2452
- });
2453
- }
2454
- if (/(?:^|\.)(?:Route|route|add_url_rule)$/.test(callable)) {
2455
- const methods = [...blockText.matchAll(/['"](GET|POST|PUT|PATCH|DELETE|OPTIONS|HEAD)['"]/g)]
2456
- .map((match) => match[1]);
2457
- const route = blockText.match(/(?:f|r|fr|rf)?['"](\/[^'"]*)['"]/)?.[1];
2458
- [...new Set(methods)].forEach((method) => addRecord(records, {
2459
- kind: 'required-inbound-operation', projections: ['external_dependencies'], presumptiveInventory: true,
2460
- path, line: index + 1, anchor: start, owner,
2461
- identity: `inbound:${method}:${route ?? callable}@${index + 1}`,
2462
- tokens: [method, route ?? callable], operationToken: method, addressToken: route,
2463
- }));
2464
- }
2465
- }
2466
- }
2467
- function scanPythonProtocolCallbacks(path, lines, owners, executable, records) {
2468
- lines.forEach((line, index) => {
2469
- if (!executable[index])
2470
- return;
2471
- const method = line.match(/^\s*(?:async\s+def|def)\s+([A-Za-z_][A-Za-z0-9_]*)\s*\(/)?.[1];
2472
- if (!method || /^__.*__$/.test(method))
2473
- return;
2474
- const owner = owners[index] ?? basename(path);
2475
- const classOwner = owner.split('.').slice(0, -1).join('.');
2476
- if (!/(?:Api|Callback|Client|Consumer|Executor|Exporter|Finder|Handler|Hook|Instrumentor|Listener|Loader|Plugin|Processor|Provider|Publisher|Receiver|Session|Transport|Wrapper)(?:\.|$)/i.test(classOwner))
2477
- return;
2478
- const projections = ['external_dependencies'];
2479
- if (/(?:Callback|Finder|Handler|Hook|Instrumentor|Loader|Plugin|Wrapper)(?:\.|$)/i.test(classOwner)) {
2480
- projections.push('stateful_resources');
2481
- }
2482
- addRecord(records, {
2483
- kind: 'protocol-callback-operation', projections, presumptiveInventory: true,
2484
- path, line: index + 1, anchor: line, owner: classOwner,
2485
- identity: `protocol-callback:${classOwner}.${method}@${index + 1}`,
2486
- tokens: [classOwner, method], operationToken: method,
2487
- });
2488
- });
2489
- }
2490
- function scanPythonOptionalImports(path, lines, owners, executable, records) {
2491
- lines.forEach((line, index) => {
2492
- if (!executable[index])
2493
- return;
2494
- const imported = line.match(/^\s*(?:from\s+([A-Za-z_][A-Za-z0-9_.]*)\s+import|import\s+([A-Za-z_][A-Za-z0-9_.]*))/);
2495
- if (!imported)
2496
- return;
2497
- const owner = owners[index] ?? basename(path);
2498
- const nearby = lines.slice(Math.max(0, index - 3), Math.min(lines.length, index + 5)).join('\n');
2499
- if (!/except\s+(?:ImportError|ModuleNotFoundError)|\btry\s*:|optional\s+(?:dependency|package)|required\s+to\s+use/i.test(nearby))
2500
- return;
2501
- const name = normalizePackageName((imported[1] ?? imported[2]).split('.')[0]);
2502
- addRecord(records, {
2503
- kind: 'optional-import-package-state', projections: ['stateful_resources'], presumptiveInventory: false,
2504
- path, line: index + 1, anchor: line, owner,
2505
- identity: `optional-import:${name}@${index + 1}`,
2506
- tokens: [name], addressToken: name, packageName: name,
2507
- });
2508
- addRecord(records, {
2509
- kind: 'optional-import-operation', projections: ['external_dependencies'], presumptiveInventory: false,
2510
- path, line: index + 1, anchor: line, owner,
2511
- identity: `optional-import:${name}@${index + 1}`,
2512
- tokens: [name], operationToken: 'import-optional-package', packageName: name,
2513
- });
2514
- });
2515
- }
2516
- const PYTHON_STANDARD_LIBRARY_ROOTS = new Set([
2517
- '__future__', 'abc', 'argparse', 'array', 'ast', 'asyncio', 'atexit', 'base64',
2518
- 'bisect', 'builtins', 'bz2', 'calendar', 'cmath', 'cmd', 'code', 'codecs',
2519
- 'collections', 'concurrent', 'configparser', 'contextlib', 'contextvars', 'copy',
2520
- 'csv', 'ctypes', 'dataclasses', 'datetime', 'decimal', 'difflib', 'dis', 'email',
2521
- 'enum', 'errno', 'faulthandler', 'fcntl', 'filecmp', 'fileinput', 'fnmatch',
2522
- 'fractions', 'ftplib', 'functools', 'gc', 'getopt', 'getpass', 'gettext', 'glob',
2523
- 'graphlib', 'gzip', 'hashlib', 'heapq', 'hmac', 'html', 'http', 'imaplib',
2524
- 'importlib', 'inspect', 'io', 'ipaddress', 'itertools', 'json', 'keyword',
2525
- 'linecache', 'locale', 'logging', 'lzma', 'mailbox', 'math', 'mimetypes',
2526
- 'multiprocessing', 'netrc', 'numbers', 'operator', 'os', 'pathlib', 'pickle',
2527
- 'pkgutil', 'platform', 'plistlib', 'pprint', 'profile', 'pstats', 'pty', 'pwd',
2528
- 'queue', 'random', 're', 'readline', 'resource', 'runpy', 'sched', 'secrets',
2529
- 'select', 'selectors', 'shelve', 'shlex', 'shutil', 'signal', 'site', 'smtplib',
2530
- 'socket', 'sqlite3', 'ssl', 'stat', 'statistics', 'string', 'stringprep',
2531
- 'struct', 'subprocess', 'sys', 'sysconfig', 'tarfile', 'tempfile', 'textwrap',
2532
- 'threading', 'time', 'timeit', 'tkinter', 'token', 'tokenize', 'tomllib',
2533
- 'trace', 'traceback', 'tracemalloc', 'types', 'typing', 'unittest', 'urllib',
2534
- 'uuid', 'venv', 'warnings', 'wave', 'weakref', 'webbrowser', 'xml', 'xmlrpc',
2535
- 'zipfile', 'zipimport', 'zlib', 'zoneinfo',
2536
- ]);
2537
- function scanPythonImportedSdkOperations(path, lines, owners, executable, localModuleRoots, records) {
2538
- const bindings = new Map();
2539
- for (const line of lines) {
2540
- const importedModule = line.match(/^\s*import\s+([A-Za-z_][A-Za-z0-9_.]*)(?:\s+as\s+([A-Za-z_][A-Za-z0-9_]*))?/);
2541
- if (importedModule) {
2542
- const root = importedModule[1].split('.')[0];
2543
- if (!PYTHON_STANDARD_LIBRARY_ROOTS.has(root) && !localModuleRoots.has(root)) {
2544
- bindings.set(importedModule[2] ?? root, importedModule[1]);
2545
- }
2546
- }
2547
- const importedSymbols = line.match(/^\s*from\s+([A-Za-z_][A-Za-z0-9_.]*)\s+import\s+(.+)$/);
2548
- if (!importedSymbols)
2549
- continue;
2550
- const root = importedSymbols[1].split('.')[0];
2551
- if (PYTHON_STANDARD_LIBRARY_ROOTS.has(root) || localModuleRoots.has(root))
2552
- continue;
2553
- for (const rawSymbol of importedSymbols[2].replace(/[()]/g, '').split(',')) {
2554
- const symbol = rawSymbol.trim().match(/^([A-Za-z_][A-Za-z0-9_]*)(?:\s+as\s+([A-Za-z_][A-Za-z0-9_]*))?/);
2555
- if (symbol)
2556
- bindings.set(symbol[2] ?? symbol[1], `${importedSymbols[1]}.${symbol[1]}`);
2557
- }
2558
- }
2559
- if (bindings.size === 0)
2560
- return;
2561
- const instanceBindings = new Map();
2562
- for (const line of lines) {
2563
- const assignment = line.match(/^\s*([A-Za-z_][A-Za-z0-9_]*)\s*=\s*([A-Za-z_][A-Za-z0-9_.]*)\s*\(/);
2564
- if (!assignment)
2565
- continue;
2566
- const root = assignment[2].split('.')[0];
2567
- const importedIdentity = bindings.get(root);
2568
- const constructor = assignment[2].split('.').at(-1) ?? '';
2569
- if (importedIdentity &&
2570
- /(?:Api|Browser|Channel|Client|Connection|Consumer|Exporter|Producer|Provider|Publisher|Reader|Resolver|Session|Stub|Transport|Writer)$/i.test(constructor)) {
2571
- instanceBindings.set(assignment[1], `${importedIdentity}${assignment[2].slice(root.length)}`);
2572
- }
2573
- }
2574
- lines.forEach((line, index) => {
2575
- if (!executable[index] || /^\s*(?:from|import)\s/.test(line))
2576
- return;
2577
- const owner = owners[index] ?? basename(path);
2578
- for (const [binding, importedIdentity] of bindings) {
2579
- const escaped = binding.replace(/[.*+?^${}()|[\]\\]/g, '\\$&');
2580
- const callPattern = new RegExp(`\\b(${escaped}(?:\\.[A-Za-z_][A-Za-z0-9_]*)*)\\s*\\(`, 'g');
2581
- for (const match of line.matchAll(callPattern)) {
2582
- const callable = match[1];
2583
- const boundaryLike = /(?:api|browser|client|convert|display|entry.?point|export|fusion|host|http|loader|model|network|plugin|resource|schema|sdk|session|socket|trait|transport|widget)/i
2584
- .test(`${importedIdentity} ${callable}`);
2585
- addRecord(records, {
2586
- kind: 'imported-sdk-operation',
2587
- projections: ['external_dependencies'],
2588
- presumptiveInventory: boundaryLike,
2589
- path,
2590
- line: index + 1,
2591
- anchor: line,
2592
- owner,
2593
- identity: `imported-sdk:${importedIdentity}:${callable}@${index + 1}:${match.index ?? 0}`,
2594
- tokens: [importedIdentity, callable],
2595
- operationToken: callable,
2596
- addressToken: importedIdentity,
2597
- });
2598
- }
2599
- }
2600
- for (const [binding, constructorIdentity] of instanceBindings) {
2601
- const escaped = binding.replace(/[.*+?^${}()|[\]\\]/g, '\\$&');
2602
- const callPattern = new RegExp(`\\b(${escaped}\\.([A-Za-z_][A-Za-z0-9_]*))\\s*\\(`, 'g');
2603
- for (const match of line.matchAll(callPattern)) {
2604
- const callable = match[1];
2605
- const method = match[2];
2606
- const boundaryLike = /^(?:ack|call|connect|consume|create|delete|execute|export|get|invoke|list|load|post|publish|put|query|read|receive|remove|request|respond|send|stream|update|write)$/i
2607
- .test(method);
2608
- addRecord(records, {
2609
- kind: 'imported-sdk-instance-operation',
2610
- projections: ['external_dependencies'],
2611
- presumptiveInventory: boundaryLike,
2612
- path,
2613
- line: index + 1,
2614
- anchor: line,
2615
- owner,
2616
- identity: `imported-sdk-instance:${constructorIdentity}:${callable}@${index + 1}:${match.index ?? 0}`,
2617
- tokens: [constructorIdentity, callable],
2618
- operationToken: callable,
2619
- addressToken: constructorIdentity,
2620
- operationIdentityKey: `${path}#${owner}:${callable}@${index + 1}`,
2621
- });
2622
- }
2623
- }
2624
- });
2625
- }
2626
- function productPublicationWorkflowJobs(path, lines) {
2627
- if (structuredPurposeTokens(basename(path)).some((token) => EXCLUDED_DIRECTORY_PURPOSE_TOKENS.has(token) || token === 'documentation'))
2628
- return new Set();
2629
- const jobs = new Map();
2630
- let jobsIndent = null;
2631
- let jobIndent = null;
2632
- let currentJob = null;
2633
- lines.forEach((line) => {
2634
- const trimmed = line.trim();
2635
- const indent = indentation(line);
2636
- if (/^jobs:\s*(?:#.*)?$/.test(trimmed)) {
2637
- jobsIndent = indent;
2638
- jobIndent = null;
2639
- currentJob = null;
2640
- return;
2641
- }
2642
- if (jobsIndent === null)
2643
- return;
2644
- if (trimmed && indent <= jobsIndent) {
2645
- jobsIndent = null;
2646
- currentJob = null;
2647
- return;
2648
- }
2649
- const job = line.match(/^\s*([A-Za-z0-9_.-]+):\s*(?:#.*)?$/)?.[1];
2650
- if (job && indent > jobsIndent && (jobIndent === null || indent === jobIndent)) {
2651
- jobIndent ??= indent;
2652
- currentJob = job;
2653
- jobs.set(job, []);
2654
- return;
2655
- }
2656
- if (currentJob)
2657
- jobs.get(currentJob)?.push(line);
2658
- });
2659
- const publicationOpcode = /(?:\bnpm\s+publish\b|\bmvn(?:w|\s)[^\n]*\bdeploy\b|\bgradlew?\b[^\n]*\bpublish\b|\btwine\s+upload\b|\bdocker\s+push\b|\bgh\s+release\s+create\b|\bcargo\s+publish\b|\bgem\s+push\b|\bgoReleaser\b|uses:\s*[^\s#]*(?:publish|release|deploy|upload-pages-artifact|deploy-pages)[^\s#]*)/i;
2660
- return new Set([...jobs].filter(([, body]) => publicationOpcode.test(body.join('\n'))).map(([job]) => job));
2661
- }
2662
- function isProductPublicationCommand(command) {
2663
- return /^(?:\.\/)?(?:mvnw?|gradlew?)\b.*\b(?:deploy|publish)\b|^npm\s+publish\b|^twine\s+upload\b|^docker\s+push\b|^gh\s+release\s+create\b|^cargo\s+publish\b|^gem\s+push\b|^git\s+(?:config|add|commit|tag|push)\b/i
2664
- .test(command.trim());
2665
- }
2666
- function scanWorkflowAndRecipeOperations(path, lines, records) {
2667
- const workflow = path.startsWith('.github/workflows/') || /(?:^|\/)(?:ci|pipeline)s?\.(?:ya?ml)$/i.test(path);
2668
- if (workflow) {
2669
- const publicationJobs = productPublicationWorkflowJobs(path, lines);
2670
- let jobsIndent = null;
2671
- let jobIndent = null;
2672
- let currentJob = 'workflow';
2673
- let triggerIndent = null;
2674
- let triggerChildIndent = null;
2675
- let currentTrigger = '';
2676
- let currentScheduleEmitted = false;
2677
- let blockRunIndent = null;
2678
- const emitSecretReads = (line, index) => {
2679
- for (const secret of line.matchAll(/\$\{\{\s*secrets\.([A-Za-z_][A-Za-z0-9_]*)\s*\}\}/g)) {
2680
- addRecord(records, {
2681
- kind: publicationJobs.has(currentJob)
2682
- ? 'workflow-publication-secret-read-operation'
2683
- : 'workflow-secret-read-operation',
2684
- projections: ['external_dependencies'],
2685
- presumptiveInventory: true,
2686
- path,
2687
- line: index + 1,
2688
- anchor: line,
2689
- owner: `workflow-job:${currentJob}`,
2690
- identity: `workflow-secret:${currentJob}:${secret[1]}@${index + 1}`,
2691
- tokens: [currentJob, secret[1]],
2692
- operationToken: 'read-workflow-secret',
2693
- addressToken: secret[1],
2694
- operationIdentityKey: `${path}#jobs.${currentJob}:secrets.${secret[1]}@${index + 1}`,
2695
- });
2696
- }
2697
- };
2698
- lines.forEach((line, index) => {
2699
- const trimmed = line.trim();
2700
- const indent = indentation(line);
2701
- emitSecretReads(line, index);
2702
- if (blockRunIndent !== null) {
2703
- if (trimmed && !trimmed.startsWith('#') && indent > blockRunIndent) {
2704
- const command = trimmed.replace(/^[-@]+/, '');
2705
- const installProfileCommand = /\bpip(?:3)?\s+install\b.*(?:-r|--requirement)\b/.test(command);
2706
- const projections = ['external_dependencies'];
2707
- if (/^git\s+(?:config|add|commit|checkout|reset|update-ref)\b/.test(command)) {
2708
- projections.push('stateful_resources');
2709
- }
2710
- addRecord(records, {
2711
- kind: publicationJobs.has(currentJob) && isProductPublicationCommand(command)
2712
- ? 'workflow-publication-command-operation'
2713
- : 'workflow-command-operation',
2714
- projections,
2715
- presumptiveInventory: !installProfileCommand,
2716
- path,
2717
- line: index + 1,
2718
- anchor: line,
2719
- owner: `workflow-job:${currentJob}`,
2720
- identity: `workflow-command:${currentJob}:${command}@${index + 1}`,
2721
- tokens: [currentJob, command],
2722
- operationToken: command.split(/\s+/)[0],
2723
- operationIdentityKey: `${path}#jobs.${currentJob}:run@${index + 1}`,
2724
- });
2725
- return;
2726
- }
2727
- if (trimmed && indent <= blockRunIndent)
2728
- blockRunIndent = null;
2729
- }
2730
- if (/^\s*jobs:\s*(?:#.*)?$/.test(line)) {
2731
- jobsIndent = indent;
2732
- jobIndent = null;
2733
- triggerIndent = null;
2734
- triggerChildIndent = null;
2735
- currentTrigger = '';
2736
- return;
2737
- }
2738
- if (jobsIndent !== null && trimmed && indent <= jobsIndent && !/^jobs:/.test(trimmed)) {
2739
- jobsIndent = null;
2740
- jobIndent = null;
2741
- currentJob = 'workflow';
2742
- }
2743
- if (jobsIndent !== null) {
2744
- const job = line.match(/^\s*([A-Za-z0-9_.-]+):\s*(?:#.*)?$/)?.[1];
2745
- if (job && indent > jobsIndent && (jobIndent === null || indent === jobIndent)) {
2746
- jobIndent ??= indent;
2747
- currentJob = job;
2748
- addRecord(records, {
2749
- kind: publicationJobs.has(job)
2750
- ? 'workflow-product-publication-runtime-unit'
2751
- : 'workflow-controlled-runtime-unit',
2752
- projections: ['runtime_roles'],
2753
- presumptiveInventory: true,
2754
- path,
2755
- line: index + 1,
2756
- anchor: line,
2757
- owner: 'workflow-jobs',
2758
- identity: `workflow-job:${path}#jobs.${job}`,
2759
- tokens: [path, job],
2760
- selectorKey: job,
2761
- implementationKey: `${path}#jobs.${job}`,
2762
- operationToken: 'execute-workflow-job',
2763
- addressToken: `${path}#jobs.${job}`,
2764
- operationIdentityKey: `${path}#jobs.${job}`,
2765
- });
2766
- }
2767
- }
2768
- if (/^\s*on:\s*(?:#.*)?$/.test(line)) {
2769
- triggerIndent = indent;
2770
- triggerChildIndent = null;
2771
- currentTrigger = '';
2772
- currentScheduleEmitted = false;
2773
- return;
2774
- }
2775
- if (triggerIndent !== null && trimmed && indent <= triggerIndent && !/^on:/.test(trimmed)) {
2776
- triggerIndent = null;
2777
- triggerChildIndent = null;
2778
- currentTrigger = '';
2779
- }
2780
- if (triggerIndent !== null && indent > triggerIndent) {
2781
- const trigger = line.match(/^\s*([A-Za-z0-9_.-]+):(?:\s*.*)?$/)?.[1];
2782
- if (trigger && (triggerChildIndent === null || indent === triggerChildIndent)) {
2783
- triggerChildIndent ??= indent;
2784
- currentTrigger = trigger;
2785
- currentScheduleEmitted = false;
2786
- if (trigger !== 'schedule')
2787
- addRecord(records, {
2788
- kind: publicationJobs.size > 0
2789
- ? 'workflow-publication-trigger-operation'
2790
- : 'workflow-trigger-operation',
2791
- projections: ['external_dependencies'],
2792
- presumptiveInventory: true,
2793
- path,
2794
- line: index + 1,
2795
- anchor: line,
2796
- owner: 'workflow-trigger',
2797
- identity: `workflow-trigger:${path}:${trigger}@${index + 1}`,
2798
- tokens: [trigger],
2799
- operationToken: trigger,
2800
- addressToken: `${path}#on.${trigger}`,
2801
- operationIdentityKey: `${path}#on.${trigger}@${index + 1}`,
2802
- });
2803
- }
2804
- const cron = line.match(/^\s*-\s*cron:\s*(['"]?)(.+?)\1\s*(?:#.*)?$/)?.[2];
2805
- if (currentTrigger === 'schedule' && cron) {
2806
- currentScheduleEmitted = true;
2807
- addRecord(records, {
2808
- kind: publicationJobs.size > 0
2809
- ? 'workflow-publication-trigger-operation'
2810
- : 'workflow-trigger-operation',
2811
- projections: ['external_dependencies'],
2812
- presumptiveInventory: true,
2813
- path,
2814
- line: index + 1,
2815
- anchor: line,
2816
- owner: 'workflow-trigger',
2817
- identity: `workflow-trigger:${path}:schedule:${cron}@${index + 1}`,
2818
- tokens: ['schedule', cron],
2819
- operationToken: 'schedule',
2820
- addressToken: cron,
2821
- operationIdentityKey: `${path}#on.schedule@${index + 1}`,
2822
- });
2823
- }
2824
- if (currentTrigger === 'schedule' &&
2825
- !currentScheduleEmitted &&
2826
- trigger &&
2827
- trigger !== 'schedule')
2828
- currentScheduleEmitted = true;
2829
- }
2830
- const uses = line.match(/^\s*-?\s*uses:\s*([^\s#]+)/)?.[1];
2831
- if (uses) {
2832
- addRecord(records, {
2833
- kind: publicationJobs.has(currentJob)
2834
- ? 'workflow-publication-action-operation'
2835
- : 'workflow-action-operation',
2836
- projections: ['external_dependencies'], presumptiveInventory: true,
2837
- path, line: index + 1, anchor: line, owner: `workflow-job:${currentJob}`,
2838
- identity: `workflow-action:${currentJob}:${uses}@${index + 1}`, tokens: [currentJob, uses],
2839
- operationToken: 'uses', addressToken: uses,
2840
- operationIdentityKey: `${path}#jobs.${currentJob}:uses@${index + 1}`,
2841
- });
2842
- if (publicationJobs.has(currentJob) && /(?:cache|artifact|publish|release|deploy)/i.test(uses)) {
2843
- addRecord(records, {
2844
- kind: 'workflow-publication-stage-state',
2845
- projections: ['stateful_resources'],
2846
- presumptiveInventory: true,
2847
- path,
2848
- line: index + 1,
2849
- anchor: line,
2850
- owner: `workflow-job:${currentJob}`,
2851
- identity: `publication-stage:${currentJob}:${uses}@${index + 1}`,
2852
- tokens: [currentJob, uses],
2853
- addressToken: `${path}#jobs.${currentJob}:uses@${index + 1}`,
2854
- materializationRootIdentityKey: `${path}#jobs.${currentJob}`,
2855
- reproductionOperationToken: `uses:${uses}`,
2856
- });
2857
- }
2858
- }
2859
- const run = line.match(/^\s*-?\s*run:\s*(.+)$/)?.[1];
2860
- if (run === '|' || run === '>') {
2861
- blockRunIndent = indent;
2862
- return;
2863
- }
2864
- if (run && run !== '|' && run !== '>')
2865
- addRecord(records, {
2866
- kind: publicationJobs.has(currentJob) && isProductPublicationCommand(run)
2867
- ? 'workflow-publication-command-operation'
2868
- : 'workflow-command-operation',
2869
- projections: ['external_dependencies'], presumptiveInventory: true,
2870
- path, line: index + 1, anchor: line, owner: `workflow-job:${currentJob}`,
2871
- identity: `workflow-command:${currentJob}:${run}@${index + 1}`, tokens: [currentJob, run],
2872
- operationToken: run.split(/\s+/)[0],
2873
- operationIdentityKey: `${path}#jobs.${currentJob}:run@${index + 1}`,
2874
- });
2875
- const runner = line.match(/^\s*runs-on:\s*([^#]+?)\s*$/)?.[1];
2876
- if (runner)
2877
- addRecord(records, {
2878
- kind: publicationJobs.has(currentJob)
2879
- ? 'workflow-publication-host-allocation-operation'
2880
- : 'workflow-host-allocation-operation',
2881
- projections: ['external_dependencies'],
2882
- presumptiveInventory: true,
2883
- path,
2884
- line: index + 1,
2885
- anchor: line,
2886
- owner: `workflow-job:${currentJob}`,
2887
- identity: `workflow-host:${currentJob}:${runner}@${index + 1}`,
2888
- tokens: [currentJob, runner],
2889
- operationToken: 'allocate-workflow-host',
2890
- addressToken: runner,
2891
- operationIdentityKey: `${path}#jobs.${currentJob}:runs-on@${index + 1}`,
2892
- });
2893
- });
2894
- }
2895
- if (basename(path).toLowerCase() === 'makefile') {
2896
- lines.forEach((line, index) => {
2897
- if (!/^\t\S/.test(line) || /^\t[@-]?(?:#|echo\b)/.test(line))
2898
- return;
2899
- const command = line.trim().replace(/^[@-]+/, '');
2900
- const projections = ['external_dependencies'];
2901
- if (/^(?:cp|install|mkdir|mv|rsync|sed|stubgen|tar|touch|uv\s+build)\b/.test(command)) {
2902
- projections.push('stateful_resources');
2903
- }
2904
- addRecord(records, {
2905
- kind: 'build-command-operation', projections, presumptiveInventory: true,
2906
- path, line: index + 1, anchor: line, owner: 'make-recipe',
2907
- identity: `make-command:${command}@${index + 1}`, tokens: [command], operationToken: command.split(/\s+/)[0],
2908
- });
2909
- });
2910
- }
2911
- }
2912
- const NON_PRODUCTION_OPTIONAL_GROUP_PARTS = new Set([
2913
- 'bench',
2914
- 'benchmark',
2915
- 'benchmarks',
2916
- 'benchmarking',
2917
- 'ci',
2918
- 'dev',
2919
- 'development',
2920
- 'doc',
2921
- 'docs',
2922
- 'documentation',
2923
- 'lint',
2924
- 'linting',
2925
- 'qa',
2926
- 'release',
2927
- 'test',
2928
- 'tests',
2929
- 'testing',
2930
- 'type',
2931
- 'types',
2932
- 'typing',
2933
- ]);
2934
- function executedCommandSurfaces(path, lines) {
2935
- const lowerName = basename(path).toLowerCase();
2936
- const extension = extname(lowerName);
2937
- const workflow = path.startsWith('.github/workflows/') ||
2938
- /(?:^|\/)(?:ci|pipeline)s?\.(?:ya?ml)$/i.test(path);
2939
- const surfaces = [];
2940
- let blockRunIndent = null;
2941
- lines.forEach((line, index) => {
2942
- const trimmed = line.trim();
2943
- const indent = indentation(line);
2944
- if (workflow) {
2945
- if (blockRunIndent !== null) {
2946
- if (trimmed && !trimmed.startsWith('#') && indent > blockRunIndent) {
2947
- surfaces.push({
2948
- line: index + 1,
2949
- anchor: line,
2950
- command: trimmed,
2951
- preserveWholeCommand: true,
2952
- });
2953
- return;
2954
- }
2955
- if (trimmed && indent <= blockRunIndent)
2956
- blockRunIndent = null;
2957
- }
2958
- const run = line.match(/^\s*-?\s*run:\s*(.+)$/)?.[1];
2959
- if (run === '|' || run === '>') {
2960
- blockRunIndent = indent;
2961
- return;
2962
- }
2963
- if (run)
2964
- surfaces.push({
2965
- line: index + 1,
2966
- anchor: line,
2967
- command: run,
2968
- preserveWholeCommand: true,
2969
- });
2970
- return;
2971
- }
2972
- if (lowerName === 'makefile') {
2973
- if (/^\t[@-]?(?!#|echo\b)\S/.test(line)) {
2974
- surfaces.push({
2975
- line: index + 1,
2976
- anchor: line,
2977
- command: trimmed.replace(/^[@-]+/, ''),
2978
- preserveWholeCommand: true,
2979
- });
2980
- }
2981
- return;
2982
- }
2983
- if (lowerName === 'dockerfile') {
2984
- const run = line.match(/^\s*RUN\s+(.+)$/i)?.[1];
2985
- if (run)
2986
- surfaces.push({
2987
- line: index + 1,
2988
- anchor: line,
2989
- command: run,
2990
- preserveWholeCommand: true,
2991
- });
2992
- return;
2993
- }
2994
- if (['.sh', '.bash', '.zsh', '.fish', '.ps1'].includes(extension)) {
2995
- if (trimmed && !trimmed.startsWith('#')) {
2996
- surfaces.push({
2997
- line: index + 1,
2998
- anchor: line,
2999
- command: trimmed,
3000
- preserveWholeCommand: true,
3001
- });
3002
- }
3003
- return;
3004
- }
3005
- if (/\b(?:subprocess\.(?:run|Popen|call|check_call|check_output)|os\.system|child_process\.(?:spawn|exec|execFile)|run_command|exec_command)\s*\(/.test(line))
3006
- surfaces.push({
3007
- line: index + 1,
3008
- anchor: line,
3009
- command: line,
3010
- preserveWholeCommand: false,
3011
- });
3012
- });
3013
- return surfaces;
3014
- }
3015
- function discoverPythonRequirementProfiles(root, tracked) {
3016
- const trackedSet = new Set(tracked);
3017
- const profiles = new Map();
3018
- for (const path of tracked) {
3019
- if (/^requirements(?:[._-](?:prod|production|runtime))?\.txt$/i.test(basename(path))) {
3020
- profiles.set(path, { path, installReferences: [] });
3021
- }
3022
- }
3023
- for (const sourcePath of tracked) {
3024
- if (!shouldScan(sourcePath))
3025
- continue;
3026
- let lines = null;
3027
- try {
3028
- lines = textLines(root, sourcePath);
3029
- }
3030
- catch {
3031
- lines = null;
3032
- }
3033
- if (!lines)
3034
- continue;
3035
- for (const surface of executedCommandSurfaces(sourcePath, lines)) {
3036
- for (const match of surface.command.matchAll(/\b(?:python(?:3(?:\.\d+)?)?\s+-m\s+)?pip(?:3)?\s+install\b[^;|&\n]*?(?:-r|--requirement)(?:\s+|=)(?:['"])([^'"]+)['"]/g)) {
3037
- const rawPath = (match[1] ?? '').trim();
3038
- const normalized = normalizePath(rawPath).replace(/^\.\//, '');
3039
- if (!normalized || !trackedSet.has(normalized))
3040
- continue;
3041
- const profile = profiles.get(normalized) ?? {
3042
- path: normalized,
3043
- installReferences: [],
3044
- };
3045
- const command = surface.preserveWholeCommand
3046
- ? surface.command.trim()
3047
- : match[0].trim();
3048
- if (!profile.installReferences.some((reference) => reference.path === sourcePath && reference.line === surface.line && reference.command === command))
3049
- profile.installReferences.push({
3050
- path: sourcePath,
3051
- line: surface.line,
3052
- anchor: surface.anchor,
3053
- command,
3054
- });
3055
- profiles.set(normalized, profile);
3056
- }
3057
- for (const match of surface.command.matchAll(/\b(?:python(?:3(?:\.\d+)?)?\s+-m\s+)?pip(?:3)?\s+install\b[^;|&\n]*?(?:-r|--requirement)(?:\s+|=)([^\s;'"|&]+)/g)) {
3058
- const normalized = normalizePath(match[1]).replace(/^\.\//, '');
3059
- if (!normalized || !trackedSet.has(normalized))
3060
- continue;
3061
- const profile = profiles.get(normalized) ?? {
3062
- path: normalized,
3063
- installReferences: [],
3064
- };
3065
- const command = surface.preserveWholeCommand
3066
- ? surface.command.trim()
3067
- : match[0].trim();
3068
- if (!profile.installReferences.some((reference) => reference.path === sourcePath && reference.line === surface.line && reference.command === command))
3069
- profile.installReferences.push({
3070
- path: sourcePath,
3071
- line: surface.line,
3072
- anchor: surface.anchor,
3073
- command,
3074
- });
3075
- profiles.set(normalized, profile);
3076
- }
3077
- }
3078
- }
3079
- for (const profile of profiles.values()) {
3080
- profile.installReferences.sort((left, right) => left.path.localeCompare(right.path) || left.line - right.line || left.command.localeCompare(right.command));
3081
- }
3082
- return profiles;
3083
- }
3084
- function scanPythonRequirementProfile(profile, lines, records) {
3085
- const requirements = lines.flatMap((line, index) => {
3086
- const trimmed = line.trim();
3087
- if (!trimmed || trimmed.startsWith('#') || /^-/.test(trimmed))
3088
- return [];
3089
- const requirement = trimmed.match(/^([A-Za-z0-9_.-]+)(\[[^\]]+\])?([^;#\s]*)(?:\s*;[^#]*)?(?:\s*#.*)?$/);
3090
- if (!requirement)
3091
- return [];
3092
- return [{
3093
- line: index + 1,
3094
- anchor: line,
3095
- name: normalizePackageName(requirement[1]),
3096
- extras: requirement[2] ?? '',
3097
- version: requirement[3] || 'unversioned',
3098
- raw: `${requirement[1]}${requirement[2] ?? ''}${requirement[3] ?? ''}`,
3099
- }];
3100
- });
3101
- const reproductionOperationToken = profile.installReferences[0]?.command;
3102
- addRecord(records, {
3103
- kind: 'python-requirements-installation-profile-state',
3104
- projections: ['stateful_resources'],
3105
- presumptiveInventory: true,
3106
- path: profile.path,
3107
- line: 1,
3108
- anchor: lines[0] ?? profile.path,
3109
- owner: 'python-requirements-profile',
3110
- identity: `requirements-profile:${profile.path}`,
3111
- tokens: [profile.path, ...requirements.map((requirement) => requirement.raw)],
3112
- addressToken: profile.path,
3113
- materializationRootIdentityKey: profile.path,
3114
- reproductionOperationToken,
3115
- });
3116
- for (const requirement of requirements)
3117
- addRecord(records, {
3118
- kind: 'python-requirement-variant-state',
3119
- projections: ['stateful_resources'],
3120
- presumptiveInventory: true,
3121
- path: profile.path,
3122
- line: requirement.line,
3123
- anchor: requirement.anchor,
3124
- owner: `python-requirements-profile:${profile.path}`,
3125
- identity: `requirement:${profile.path}:${requirement.line}:${requirement.raw}`,
3126
- tokens: [profile.path, requirement.raw, requirement.name, requirement.extras, requirement.version],
3127
- addressToken: requirement.name,
3128
- packageName: requirement.name,
3129
- packageVersion: requirement.version,
3130
- materializationRootIdentityKey: profile.path,
3131
- reproductionOperationToken,
3132
- });
3133
- for (const reference of profile.installReferences)
3134
- addRecord(records, {
3135
- kind: 'python-requirements-profile-install-operation',
3136
- projections: ['external_dependencies'],
3137
- presumptiveInventory: true,
3138
- path: reference.path,
3139
- line: reference.line,
3140
- anchor: reference.anchor,
3141
- owner: 'executed-install-command',
3142
- identity: `install-profile:${profile.path}:${reference.path}:${reference.line}:${reference.command}`,
3143
- tokens: [profile.path, reference.command],
3144
- operationToken: reference.command,
3145
- addressToken: profile.path,
3146
- operationIdentityKey: `${reference.path}#install:${reference.line}:${reference.command}`,
3147
- materializationRootIdentityKey: profile.path,
3148
- reproductionOperationToken: reference.command,
3149
- });
3150
- }
3151
- function isNonProductionOptionalGroup(value) {
3152
- return value
3153
- .trim()
3154
- .toLowerCase()
3155
- .split(/[._-]+/)
3156
- .some((part) => NON_PRODUCTION_OPTIONAL_GROUP_PARTS.has(part));
3157
- }
3158
- function optionalGroupName(line) {
3159
- const match = line.trim().match(/^(?:"([^"]+)"|'([^']+)'|([A-Za-z0-9_.-]+))\s*=\s*\[/);
3160
- return match?.[1] ?? match?.[2] ?? match?.[3];
3161
- }
3162
- function artifactFilename(url) {
3163
- const path = url.split(/[?#]/, 1)[0];
3164
- const encoded = path.slice(path.lastIndexOf('/') + 1);
3165
- try {
3166
- return decodeURIComponent(encoded);
3167
- }
3168
- catch {
3169
- return encoded;
3170
- }
3171
- }
3172
- function parseUvPackages(lines) {
3173
- const starts = [];
3174
- lines.forEach((line, index) => { if (line.trim() === '[[package]]')
3175
- starts.push(index); });
3176
- return starts.flatMap((start, position) => {
3177
- const end = starts[position + 1] ?? lines.length;
3178
- const block = lines.slice(start, end);
3179
- const nameIndex = block.findIndex((line) => /^name\s*=\s*"/.test(line));
3180
- if (nameIndex < 0)
3181
- return [];
3182
- const name = block[nameIndex].match(/^name\s*=\s*"([^"]+)"/)?.[1];
3183
- if (!name)
3184
- return [];
3185
- const versionIndex = block.findIndex((line) => /^version\s*=\s*"/.test(line));
3186
- const version = versionIndex < 0 ? 'workspace' : block[versionIndex].match(/^version\s*=\s*"([^"]+)"/)?.[1] ?? 'workspace';
3187
- const sourceIndex = block.findIndex((line) => /^source\s*=/.test(line));
3188
- const sourceLine = sourceIndex < 0 ? block[nameIndex] : block[sourceIndex];
3189
- const dependencies = new Set();
3190
- const optionalDependencies = new Set();
3191
- const artifacts = [];
3192
- let section = '';
3193
- let optionalGroup = '';
3194
- for (let blockIndex = 0; blockIndex < block.length; blockIndex += 1) {
3195
- const line = block[blockIndex];
3196
- const trimmed = line.trim();
3197
- if (/^dependencies\s*=\s*\[/.test(trimmed)) {
3198
- section = 'dependencies';
3199
- optionalGroup = '';
3200
- }
3201
- else if (trimmed === '[package.optional-dependencies]') {
3202
- section = 'optional';
3203
- optionalGroup = '';
3204
- }
3205
- else if (trimmed.startsWith('[package.dev-dependencies]') || trimmed.startsWith('[package.metadata]')) {
3206
- section = 'excluded';
3207
- optionalGroup = '';
3208
- }
3209
- else if (section === 'optional') {
3210
- const group = optionalGroupName(trimmed);
3211
- if (group)
3212
- optionalGroup = group;
3213
- }
3214
- for (const dependency of line.matchAll(/\{\s*name\s*=\s*"([^"]+)"/g)) {
3215
- if (section === 'dependencies')
3216
- dependencies.add(dependency[1]);
3217
- if (section === 'optional' &&
3218
- optionalGroup &&
3219
- !isNonProductionOptionalGroup(optionalGroup))
3220
- optionalDependencies.add(dependency[1]);
3221
- }
3222
- for (const artifact of line.matchAll(/\{[^{}]*\burl\s*=\s*"([^"]+)"[^{}]*\bhash\s*=\s*"(sha256:[0-9a-fA-F]+)"[^{}]*\}/g)) {
3223
- const url = artifact[1];
3224
- const filename = artifactFilename(url);
3225
- artifacts.push({
3226
- kind: /^sdist\s*=/.test(trimmed) || !filename.endsWith('.whl') ? 'sdist' : 'wheel',
3227
- filename,
3228
- url,
3229
- hash: artifact[2].toLowerCase(),
3230
- line: start + blockIndex + 1,
3231
- anchor: line,
3232
- });
3233
- }
3234
- }
3235
- return [{
3236
- name,
3237
- version,
3238
- line: start + nameIndex + 1,
3239
- sourceLine,
3240
- editable: /editable\s*=|virtual\s*=/.test(sourceLine),
3241
- dependencies,
3242
- optionalDependencies,
3243
- artifacts,
3244
- }];
3245
- });
3246
- }
3247
- function scanUvLock(path, lines, records) {
3248
- const packages = parseUvPackages(lines);
3249
- const byName = new Map();
3250
- for (const pkg of packages) {
3251
- const variants = byName.get(pkg.name) ?? [];
3252
- variants.push(pkg);
3253
- byName.set(pkg.name, variants);
3254
- }
3255
- const roots = packages.filter((pkg) => pkg.editable);
3256
- const rootNames = new Set(roots.map((pkg) => pkg.name));
3257
- const queue = roots.flatMap((pkg) => [...pkg.dependencies, ...pkg.optionalDependencies]);
3258
- const reachable = new Set();
3259
- while (queue.length > 0) {
3260
- const name = queue.shift();
3261
- if (rootNames.has(name))
3262
- continue;
3263
- if (reachable.has(name))
3264
- continue;
3265
- const variants = byName.get(name);
3266
- if (!variants)
3267
- continue;
3268
- reachable.add(name);
3269
- for (const pkg of variants)
3270
- queue.push(...pkg.dependencies);
3271
- }
3272
- for (const name of [...reachable].sort()) {
3273
- for (const pkg of byName.get(name)) {
3274
- if (pkg.artifacts.length > 0) {
3275
- for (const artifact of pkg.artifacts) {
3276
- const tokens = [
3277
- name,
3278
- pkg.version,
3279
- artifact.kind,
3280
- artifact.filename,
3281
- artifact.url,
3282
- artifact.hash,
3283
- ];
3284
- const variantIdentity = `${name}@${pkg.version}:${artifact.kind}:${artifact.hash}:${artifact.url}`;
3285
- addRecord(records, {
3286
- kind: 'locked-production-package-artifact-state',
3287
- projections: ['stateful_resources'],
3288
- presumptiveInventory: true,
3289
- path,
3290
- line: artifact.line,
3291
- anchor: artifact.anchor,
3292
- owner: 'production-package-artifact-closure',
3293
- identity: `package-artifact:${variantIdentity}`,
3294
- tokens,
3295
- addressToken: artifact.filename,
3296
- packageName: name,
3297
- packageVersion: pkg.version,
3298
- packageArtifactKind: artifact.kind,
3299
- packageArtifactFilename: artifact.filename,
3300
- packageArtifactUrl: artifact.url,
3301
- packageArtifactHash: artifact.hash,
3302
- });
3303
- addRecord(records, {
3304
- kind: 'locked-production-package-artifact-install-operation',
3305
- projections: ['external_dependencies'],
3306
- presumptiveInventory: true,
3307
- path,
3308
- line: artifact.line,
3309
- anchor: artifact.anchor,
3310
- owner: 'production-package-artifact-closure',
3311
- identity: `pull-package-artifact:${variantIdentity}`,
3312
- tokens,
3313
- operationToken: 'pull-locked-package-artifact',
3314
- addressToken: artifact.url,
3315
- packageName: name,
3316
- packageVersion: pkg.version,
3317
- packageArtifactKind: artifact.kind,
3318
- packageArtifactFilename: artifact.filename,
3319
- packageArtifactUrl: artifact.url,
3320
- packageArtifactHash: artifact.hash,
3321
- });
3322
- }
3323
- continue;
3324
- }
3325
- const anchor = lines[pkg.line - 1];
3326
- addRecord(records, {
3327
- kind: 'locked-production-package-state',
3328
- projections: ['stateful_resources'],
3329
- presumptiveInventory: true,
3330
- path, line: pkg.line, anchor, owner: 'production-package-closure',
3331
- identity: `package:${name}@${pkg.version}`, tokens: [name, pkg.version],
3332
- addressToken: name, packageName: name, packageVersion: pkg.version,
3333
- });
3334
- addRecord(records, {
3335
- kind: 'locked-production-package-install-operation',
3336
- projections: ['external_dependencies'],
3337
- presumptiveInventory: true,
3338
- path, line: pkg.line, anchor, owner: 'production-package-closure',
3339
- identity: `install:${name}@${pkg.version}`, tokens: [name, pkg.version],
3340
- operationToken: 'install-locked-package', packageName: name, packageVersion: pkg.version,
3341
- });
3342
- }
3343
- }
3344
- }
3345
- function scanPackageLock(path, text, records) {
3346
- let parsed;
3347
- try {
3348
- parsed = JSON.parse(text);
3349
- }
3350
- catch {
3351
- return;
3352
- }
3353
- if (!parsed || typeof parsed !== 'object')
3354
- return;
3355
- const lock = parsed;
3356
- const lines = text.split(/\r?\n/);
3357
- const emittedLocations = new Set();
3358
- const emitPackage = (location, packageName, pkg) => {
3359
- if (!location || pkg.dev === true || emittedLocations.has(location))
3360
- return;
3361
- emittedLocations.add(location);
3362
- const name = pkg.name ?? packageName;
3363
- const version = pkg.version ?? 'locked';
3364
- const quotedLocation = JSON.stringify(location);
3365
- const exactLocatorTokens = [
3366
- `${quotedLocation}:`,
3367
- pkg.resolved ? JSON.stringify(pkg.resolved) : '',
3368
- pkg.integrity ? JSON.stringify(pkg.integrity) : '',
3369
- JSON.stringify(packageName),
3370
- ].filter(Boolean);
3371
- const lineIndex = exactLocatorTokens
3372
- .map((token) => lines.findIndex((line) => line.includes(token)))
3373
- .find((index) => index >= 0) ?? -1;
3374
- const line = lineIndex >= 0 ? lineIndex + 1 : 1;
3375
- const anchor = lines[line - 1] ?? '{';
3376
- const exactArtifact = Boolean(pkg.resolved && pkg.integrity);
3377
- const filename = pkg.resolved ? artifactFilename(pkg.resolved) : undefined;
3378
- for (const [kind, projection, operationToken] of [
3379
- [exactArtifact ? 'locked-production-package-artifact-state' : 'locked-production-package-state', 'stateful_resources', undefined],
3380
- [exactArtifact ? 'locked-production-package-artifact-install-operation' : 'locked-production-package-install-operation', 'external_dependencies', exactArtifact ? 'pull-locked-package-artifact' : 'install-locked-package'],
3381
- ])
3382
- addRecord(records, {
3383
- kind, projections: [projection], presumptiveInventory: true,
3384
- path, line, anchor, owner: 'production-package-closure',
3385
- identity: `${operationToken ? 'install' : 'package'}:${name}@${version}:${location}:${pkg.resolved ?? ''}:${pkg.integrity ?? ''}`,
3386
- tokens: [name, version, location, pkg.resolved ?? '', pkg.integrity ?? ''],
3387
- packageName: name,
3388
- packageVersion: version,
3389
- ...(exactArtifact ? {
3390
- packageArtifactKind: 'tarball',
3391
- packageArtifactFilename: filename,
3392
- packageArtifactUrl: pkg.resolved,
3393
- packageArtifactHash: pkg.integrity,
3394
- } : {}),
3395
- ...(operationToken ? { operationToken, addressToken: pkg.resolved } : { addressToken: filename ?? name }),
3396
- });
3397
- };
3398
- if (lock.packages) {
3399
- Object.entries(lock.packages).forEach(([location, pkg]) => {
3400
- if (!location)
3401
- return;
3402
- const name = pkg.name ?? location.split('/node_modules/').at(-1)
3403
- .replace(/^node_modules\//, '');
3404
- emitPackage(location, name, pkg);
3405
- });
3406
- return;
3407
- }
3408
- const walkLegacy = (dependencies, parentLocation = '') => {
3409
- Object.entries(dependencies).forEach(([name, pkg]) => {
3410
- if (pkg.dev === true)
3411
- return;
3412
- const location = parentLocation
3413
- ? `${parentLocation}/node_modules/${name}`
3414
- : `node_modules/${name}`;
3415
- emitPackage(location, name, pkg);
3416
- if (pkg.dependencies)
3417
- walkLegacy(pkg.dependencies, location);
3418
- });
3419
- };
3420
- if (lock.dependencies)
3421
- walkLegacy(lock.dependencies);
3422
- }
3423
- function scanPythonBuildRequirements(path, lines, records) {
3424
- let inBuild = false;
3425
- let inRequires = false;
3426
- lines.forEach((line, index) => {
3427
- const trimmed = line.trim();
3428
- if (/^\[.+\]$/.test(trimmed)) {
3429
- inBuild = trimmed === '[build-system]';
3430
- inRequires = false;
3431
- return;
3432
- }
3433
- if (!inBuild)
3434
- return;
3435
- if (/^requires\s*=\s*\[/.test(trimmed))
3436
- inRequires = true;
3437
- if (!inRequires)
3438
- return;
3439
- for (const match of line.matchAll(/['"]([A-Za-z0-9_.-]+)(?:\[[^\]]+\])?(?:\s*[^'"]*)?['"]/g)) {
3440
- const name = match[1].toLowerCase().replace(/_/g, '-');
3441
- addRecord(records, {
3442
- kind: 'build-system-package-state', projections: ['stateful_resources'], presumptiveInventory: true,
3443
- path, line: index + 1, anchor: line, owner: 'build-system', identity: `package:${name}`,
3444
- tokens: [name], addressToken: name, packageName: name,
3445
- });
3446
- addRecord(records, {
3447
- kind: 'build-system-package-install-operation', projections: ['external_dependencies'], presumptiveInventory: true,
3448
- path, line: index + 1, anchor: line, owner: 'build-system', identity: `install:${name}`,
3449
- tokens: [name], operationToken: 'install-build-system-package', packageName: name,
3450
- });
3451
- }
3452
- if (trimmed.includes(']'))
3453
- inRequires = false;
3454
- });
3455
- }
3456
- function normalizePackageName(value) {
3457
- return value.trim().toLowerCase().replace(/_/g, '-');
3458
- }
3459
- function xmlBlocks(source, tag) {
3460
- const escaped = tag.replace(/[.*+?^${}()|[\]\\]/g, '\\$&');
3461
- const pattern = new RegExp(`<${escaped}(?:\\s[^>]*)?>([\\s\\S]*?)<\\/${escaped}>`, 'g');
3462
- const blocks = [];
3463
- for (const match of source.matchAll(pattern)) {
3464
- const start = match.index ?? 0;
3465
- blocks.push({
3466
- start,
3467
- end: start + match[0].length,
3468
- line: source.slice(0, start).split('\n').length,
3469
- text: match[1],
3470
- });
3471
- }
3472
- return blocks;
3473
- }
3474
- function xmlValue(source, tag) {
3475
- const escaped = tag.replace(/[.*+?^${}()|[\]\\]/g, '\\$&');
3476
- return source.match(new RegExp(`<${escaped}(?:\\s[^>]*)?>\\s*([^<]+?)\\s*<\\/${escaped}>`))?.[1].trim();
3477
- }
3478
- function structuredPurposeTokens(value) {
3479
- return value
3480
- .replace(/([a-z0-9])([A-Z])/g, '$1-$2')
3481
- .toLowerCase()
3482
- .split(/[^a-z0-9]+/)
3483
- .filter(Boolean);
3484
- }
3485
- const NON_PRODUCTION_MAVEN_SCOPE_TOKENS = new Set([
3486
- 'benchmark', 'benchmarks', 'checkstyle', 'coverage', 'dev', 'docs', 'integration',
3487
- 'documenter', 'failsafe', 'format', 'jacoco', 'javaformat', 'jmh', 'lint', 'qa', 'site', 'spec',
3488
- 'surefire', 'test', 'tests', 'verification', 'versions',
3489
- ]);
3490
- function isNonProductionMavenUnit(...values) {
3491
- return values.some((value) => value !== undefined && structuredPurposeTokens(value)
3492
- .some((token) => NON_PRODUCTION_MAVEN_SCOPE_TOKENS.has(token)));
3493
- }
3494
- function scanMavenManifest(path, lines, records) {
3495
- if (basename(path).toLowerCase() !== 'pom.xml')
3496
- return;
3497
- const source = lines.join('\n');
3498
- const profiles = xmlBlocks(source, 'profile').map((block) => ({
3499
- ...block,
3500
- id: xmlValue(block.text, 'id') ?? `profile@${block.line}`,
3501
- }));
3502
- const profileAt = (offset) => profiles.find((profile) => offset >= profile.start && offset < profile.end)?.id;
3503
- const lineAnchor = (line) => lines[line - 1] ?? '<project>';
3504
- const beforeBuildSections = source
3505
- .replace(/<parent(?:\s[^>]*)?>[\s\S]*?<\/parent>/, '')
3506
- .split(/<(?:dependencies|dependencyManagement|build|profiles|modules)(?:\s|>)/, 1)[0];
3507
- const groupId = xmlValue(beforeBuildSections, 'groupId') ?? 'inherited-group';
3508
- const artifactId = xmlValue(beforeBuildSections, 'artifactId')
3509
- ?? (normalizePath(path).replace(/\/pom\.xml$/, '') || 'root-project');
3510
- const version = xmlValue(beforeBuildSections, 'version') ?? 'inherited-version';
3511
- const packaging = xmlValue(beforeBuildSections, 'packaging') ?? 'jar';
3512
- const moduleIdentity = `${path}#maven-project:${groupId}:${artifactId}:${version}`;
3513
- if (packaging !== 'pom')
3514
- addRecord(records, {
3515
- kind: 'maven-product-distribution-state',
3516
- projections: ['stateful_resources'],
3517
- presumptiveInventory: true,
3518
- path,
3519
- line: 1,
3520
- anchor: lines.find((line) => line.includes('<artifactId>')) ?? lines[0] ?? '<project>',
3521
- owner: 'maven-project',
3522
- identity: `distribution:${groupId}:${artifactId}:${version}:${packaging}`,
3523
- tokens: [groupId, artifactId, version, packaging],
3524
- addressToken: `${groupId}:${artifactId}`,
3525
- packageName: `${groupId}:${artifactId}`,
3526
- packageVersion: version,
3527
- packageArtifactKind: packaging,
3528
- materializationRootIdentityKey: moduleIdentity,
3529
- reproductionOperationToken: 'mvn package',
3530
- });
3531
- profiles.filter((profile) => !isNonProductionMavenUnit(profile.id)).forEach((profile) => addRecord(records, {
3532
- kind: 'maven-installation-profile-state',
3533
- projections: ['stateful_resources'],
3534
- presumptiveInventory: true,
3535
- path,
3536
- line: profile.line,
3537
- anchor: lineAnchor(profile.line),
3538
- owner: 'maven-profiles',
3539
- identity: `profile:${moduleIdentity}:${profile.id}`,
3540
- tokens: [moduleIdentity, profile.id],
3541
- selectorKey: profile.id,
3542
- addressToken: `${groupId}:${artifactId}:${profile.id}`,
3543
- materializationRootIdentityKey: `${moduleIdentity}:profile:${profile.id}`,
3544
- reproductionOperationToken: `mvn -P${profile.id} package`,
3545
- }));
3546
- const pluginBlocks = xmlBlocks(source, 'plugin');
3547
- const enclosingPlugin = (block) => pluginBlocks.find((plugin) => block.start > plugin.start && block.end < plugin.end);
3548
- xmlBlocks(source, 'dependency').forEach((dependency) => {
3549
- const dependencyGroup = xmlValue(dependency.text, 'groupId') ?? 'inherited-group';
3550
- const dependencyArtifact = xmlValue(dependency.text, 'artifactId');
3551
- if (!dependencyArtifact)
3552
- return;
3553
- const dependencyVersion = xmlValue(dependency.text, 'version') ?? 'managed-version';
3554
- const scope = xmlValue(dependency.text, 'scope') ?? 'compile';
3555
- if (scope === 'test' || isNonProductionMavenUnit(scope))
3556
- return;
3557
- const profile = profileAt(dependency.start) ?? 'base';
3558
- const plugin = enclosingPlugin(dependency);
3559
- const type = xmlValue(dependency.text, 'type') ?? 'jar';
3560
- const classifier = xmlValue(dependency.text, 'classifier') ?? 'default';
3561
- const optional = xmlValue(dependency.text, 'optional') ?? 'false';
3562
- const coordinate = `${dependencyGroup}:${dependencyArtifact}:${type}:${classifier}:${dependencyVersion}`;
3563
- addRecord(records, {
3564
- kind: plugin ? 'maven-plugin-dependency-requirement-state' : 'maven-dependency-requirement-state',
3565
- projections: ['stateful_resources'],
3566
- presumptiveInventory: true,
3567
- path,
3568
- line: dependency.line,
3569
- anchor: lineAnchor(dependency.line),
3570
- owner: `maven-profile:${profile}`,
3571
- identity: `requirement:${moduleIdentity}:${profile}:${scope}:${coordinate}:optional=${optional}`,
3572
- tokens: [moduleIdentity, profile, scope, coordinate, optional],
3573
- addressToken: coordinate,
3574
- packageName: `${dependencyGroup}:${dependencyArtifact}`,
3575
- packageVersion: dependencyVersion,
3576
- packageArtifactKind: type,
3577
- materializationRootIdentityKey: `${moduleIdentity}:profile:${profile}`,
3578
- reproductionOperationToken: profile === 'base' ? 'mvn package' : `mvn -P${profile} package`,
3579
- });
3580
- });
3581
- pluginBlocks.forEach((plugin) => {
3582
- const pluginGroup = xmlValue(plugin.text, 'groupId') ?? 'org.apache.maven.plugins';
3583
- const pluginArtifact = xmlValue(plugin.text, 'artifactId');
3584
- if (!pluginArtifact)
3585
- return;
3586
- const pluginVersion = xmlValue(plugin.text, 'version') ?? 'managed-version';
3587
- const profile = profileAt(plugin.start) ?? 'base';
3588
- const pluginCoordinate = `${pluginGroup}:${pluginArtifact}:${pluginVersion}`;
3589
- if (!isNonProductionMavenUnit(pluginArtifact))
3590
- addRecord(records, {
3591
- kind: 'maven-build-plugin-requirement-state',
3592
- projections: ['stateful_resources'],
3593
- presumptiveInventory: true,
3594
- path,
3595
- line: plugin.line,
3596
- anchor: lineAnchor(plugin.line),
3597
- owner: `maven-profile:${profile}`,
3598
- identity: `plugin:${moduleIdentity}:${profile}:${pluginCoordinate}`,
3599
- tokens: [moduleIdentity, profile, pluginCoordinate],
3600
- addressToken: pluginCoordinate,
3601
- packageName: `${pluginGroup}:${pluginArtifact}`,
3602
- packageVersion: pluginVersion,
3603
- packageArtifactKind: 'maven-plugin',
3604
- materializationRootIdentityKey: `${moduleIdentity}:profile:${profile}`,
3605
- reproductionOperationToken: profile === 'base' ? 'mvn package' : `mvn -P${profile} package`,
3606
- });
3607
- const executions = xmlBlocks(plugin.text, 'execution');
3608
- const admittedExecutions = executions.map((execution) => ({
3609
- ...execution,
3610
- absoluteLine: plugin.line + execution.line - 1,
3611
- id: xmlValue(execution.text, 'id') ?? `execution@${plugin.line + execution.line - 1}`,
3612
- phase: xmlValue(execution.text, 'phase'),
3613
- goals: xmlBlocks(execution.text, 'goal').map((goal) => goal.text.trim()).filter(Boolean),
3614
- })).filter((execution) => !isNonProductionMavenUnit(pluginArtifact, execution.id, execution.phase, ...execution.goals));
3615
- if (admittedExecutions.length === 0)
3616
- return;
3617
- const familyIdentity = `plugin-executions:${moduleIdentity}:${profile}:${pluginCoordinate}`;
3618
- addRecord(records, {
3619
- kind: 'maven-plugin-execution-family',
3620
- projections: ['registry_families'],
3621
- presumptiveInventory: true,
3622
- path,
3623
- line: plugin.line,
3624
- anchor: lineAnchor(plugin.line),
3625
- owner: `maven-plugin:${pluginCoordinate}`,
3626
- identity: familyIdentity,
3627
- tokens: [pluginCoordinate, profile],
3628
- implementationKey: pluginCoordinate,
3629
- });
3630
- admittedExecutions.forEach((execution) => addRecord(records, {
3631
- kind: 'maven-plugin-execution-entrypoint-state',
3632
- projections: ['stateful_resources', 'registry_entries'],
3633
- presumptiveInventory: true,
3634
- path,
3635
- line: execution.absoluteLine,
3636
- anchor: lineAnchor(execution.absoluteLine),
3637
- owner: `maven-plugin:${pluginCoordinate}`,
3638
- identity: `${familyIdentity}:${execution.id}`,
3639
- tokens: [pluginCoordinate, execution.id, execution.phase ?? '', ...execution.goals],
3640
- selectorKey: execution.id,
3641
- implementationKey: `${pluginCoordinate}:${execution.goals.join('+') || 'default-goal'}`,
3642
- addressToken: execution.id,
3643
- materializationRootIdentityKey: `${moduleIdentity}:profile:${profile}`,
3644
- reproductionOperationToken: profile === 'base'
3645
- ? `mvn ${pluginGroup}:${pluginArtifact}:${execution.goals[0] ?? execution.id}`
3646
- : `mvn -P${profile} ${pluginGroup}:${pluginArtifact}:${execution.goals[0] ?? execution.id}`,
3647
- }));
3648
- });
3649
- }
3650
- function addDeclaredProductionPackage(path, line, anchor, rawName, version, profileIdentityKey, records) {
3651
- const name = normalizePackageName(rawName);
3652
- if (!name || name === 'python')
3653
- return;
3654
- const alreadyDeclared = [...records.values()].some((record) => record.kind === 'declared-production-package-state' &&
3655
- record.package_name === name &&
3656
- record.package_version === version &&
3657
- record.materialization_root_identity_key === profileIdentityKey);
3658
- if (alreadyDeclared)
3659
- return;
3660
- addRecord(records, {
3661
- kind: 'declared-production-package-state', projections: ['stateful_resources'], presumptiveInventory: true,
3662
- path, line, anchor, owner: `production-package-profile:${profileIdentityKey}`,
3663
- identity: `package:${profileIdentityKey}:${name}@${version}`, tokens: [profileIdentityKey, name, version],
3664
- addressToken: name, packageName: name, packageVersion: version,
3665
- materializationRootIdentityKey: profileIdentityKey,
3666
- });
3667
- addRecord(records, {
3668
- kind: 'declared-production-package-install-operation', projections: ['external_dependencies'], presumptiveInventory: true,
3669
- path, line, anchor, owner: `production-package-profile:${profileIdentityKey}`,
3670
- identity: `install:${profileIdentityKey}:${name}@${version}`, tokens: [profileIdentityKey, name, version],
3671
- operationToken: 'install-production-package', packageName: name, packageVersion: version,
3672
- materializationRootIdentityKey: profileIdentityKey,
3673
- operationIdentityKey: `${profileIdentityKey}:install:${name}@${version}`,
3674
- });
3675
- }
3676
- function scanProductionManifestPackages(path, lines, records) {
3677
- const lowerName = basename(path).toLowerCase();
3678
- if (lowerName === 'pyproject.toml') {
3679
- let projectSection = false;
3680
- let projectName = 'python-project';
3681
- for (const line of lines) {
3682
- const header = line.trim().match(/^\[([^\]]+)\]$/)?.[1];
3683
- if (header) {
3684
- projectSection = header === 'project';
3685
- continue;
3686
- }
3687
- if (!projectSection)
3688
- continue;
3689
- const declaredName = line.trim().match(/^name\s*=\s*['"]([^'"]+)['"]/)?.[1];
3690
- if (declaredName) {
3691
- projectName = normalizePackageName(declaredName);
3692
- break;
3693
- }
3694
- }
3695
- let section = '';
3696
- let inProjectArray = false;
3697
- let optionalGroup = '';
3698
- let inOptionalArray = false;
3699
- lines.forEach((line, index) => {
3700
- const trimmed = line.trim();
3701
- const header = trimmed.match(/^\[([^\]]+)\]$/)?.[1];
3702
- if (header) {
3703
- section = header;
3704
- inProjectArray = false;
3705
- inOptionalArray = false;
3706
- optionalGroup = '';
3707
- if (header === 'project')
3708
- addRecord(records, {
3709
- kind: 'python-installation-profile-state',
3710
- projections: ['stateful_resources'],
3711
- presumptiveInventory: true,
3712
- path,
3713
- line: index + 1,
3714
- anchor: line,
3715
- owner: 'python-installation-profile',
3716
- identity: `install-profile:${projectName}`,
3717
- tokens: [projectName],
3718
- addressToken: projectName,
3719
- packageName: projectName,
3720
- packageVersion: 'base-profile',
3721
- });
3722
- return;
3723
- }
3724
- const projectArray = section === 'project' && /^dependencies\s*=\s*\[/.test(trimmed);
3725
- if (projectArray) {
3726
- inProjectArray = true;
3727
- }
3728
- if (section === 'project.optional-dependencies') {
3729
- const group = optionalGroupName(trimmed);
3730
- if (group) {
3731
- optionalGroup = group;
3732
- inOptionalArray = true;
3733
- if (!isNonProductionOptionalGroup(group))
3734
- addRecord(records, {
3735
- kind: 'python-installation-profile-state',
3736
- projections: ['stateful_resources'],
3737
- presumptiveInventory: true,
3738
- path,
3739
- line: index + 1,
3740
- anchor: line,
3741
- owner: 'python-installation-profile',
3742
- identity: `install-profile:${projectName}[${group}]`,
3743
- tokens: [projectName, group],
3744
- addressToken: `${projectName}[${group}]`,
3745
- packageName: projectName,
3746
- packageVersion: `extra:${group}`,
3747
- });
3748
- }
3749
- }
3750
- const includeProjectLine = projectArray || (section === 'project' && inProjectArray);
3751
- const includeOptionalLine = section === 'project.optional-dependencies' &&
3752
- inOptionalArray &&
3753
- optionalGroup.length > 0 &&
3754
- !isNonProductionOptionalGroup(optionalGroup);
3755
- const includePoetryLine = section === 'tool.poetry.dependencies';
3756
- if (includeProjectLine || includeOptionalLine) {
3757
- const profileIdentityKey = includeOptionalLine
3758
- ? `${path}#project.optional-dependencies.${optionalGroup}`
3759
- : `${path}#project`;
3760
- for (const match of line.matchAll(/['"]([A-Za-z0-9_.-]+)(?:\[[^\]]+\])?([^'"]*)['"]/g)) {
3761
- if (normalizePackageName(match[1]) === projectName)
3762
- continue;
3763
- addDeclaredProductionPackage(path, index + 1, line, match[1], match[2].trim() || 'declared', profileIdentityKey, records);
3764
- }
3765
- }
3766
- else if (includePoetryLine) {
3767
- const dependency = line.match(/^\s*([A-Za-z0-9_.-]+)\s*=\s*(.+)$/);
3768
- if (dependency)
3769
- addDeclaredProductionPackage(path, index + 1, line, dependency[1], dependency[2].trim(), `${path}#tool.poetry.dependencies`, records);
3770
- }
3771
- if (includeProjectLine && trimmed.includes(']'))
3772
- inProjectArray = false;
3773
- if (includeOptionalLine && trimmed.includes(']'))
3774
- inOptionalArray = false;
3775
- });
3776
- }
3777
- else if (lowerName === 'package.json') {
3778
- let parsed;
3779
- try {
3780
- parsed = JSON.parse(lines.join('\n'));
3781
- }
3782
- catch {
3783
- return;
3784
- }
3785
- const dependencies = { ...(parsed.dependencies ?? {}), ...(parsed.optionalDependencies ?? {}) };
3786
- Object.entries(dependencies).forEach(([name, version]) => {
3787
- const lineIndex = lines.findIndex((line) => line.includes(JSON.stringify(name)));
3788
- const line = lineIndex >= 0 ? lineIndex + 1 : 1;
3789
- addDeclaredProductionPackage(path, line, lines[line - 1] ?? '{', name, version, `${path}#dependencies`, records);
3790
- });
3791
- }
3792
- else if (lowerName === 'cargo.toml') {
3793
- let productionSection = false;
3794
- let productionSectionIdentity = '';
3795
- lines.forEach((line, index) => {
3796
- const header = line.trim().match(/^\[([^\]]+)\]$/)?.[1];
3797
- if (header) {
3798
- productionSection = /(?:^|\.)(?:dependencies|build-dependencies)$/.test(header) && !/(?:^|\.)dev-dependencies$/.test(header);
3799
- productionSectionIdentity = productionSection ? `${path}#${header}` : '';
3800
- return;
3801
- }
3802
- if (!productionSection)
3803
- return;
3804
- const dependency = line.match(/^\s*([A-Za-z0-9_.-]+)\s*=\s*(.+)$/);
3805
- if (dependency)
3806
- addDeclaredProductionPackage(path, index + 1, line, dependency[1], dependency[2].trim(), productionSectionIdentity, records);
3807
- });
3808
- }
3809
- else if (lowerName === 'go.mod') {
3810
- let requireBlock = false;
3811
- lines.forEach((line, index) => {
3812
- const trimmed = line.trim();
3813
- if (trimmed === 'require (') {
3814
- requireBlock = true;
3815
- return;
3816
- }
3817
- if (requireBlock && trimmed === ')') {
3818
- requireBlock = false;
3819
- return;
3820
- }
3821
- const match = (requireBlock ? trimmed : trimmed.replace(/^require\s+/, '')).match(/^([^\s]+)\s+([^\s]+)/);
3822
- if ((requireBlock || trimmed.startsWith('require ')) && match) {
3823
- addDeclaredProductionPackage(path, index + 1, line, match[1], match[2], `${path}#require`, records);
3824
- }
3825
- });
3826
- }
3827
- }
3828
- function scanManifestEntrypoints(path, lines, records) {
3829
- const lowerName = basename(path).toLowerCase();
3830
- if (lowerName === 'pom.xml')
3831
- scanMavenManifest(path, lines, records);
3832
- if (lowerName === 'pyproject.toml') {
3833
- let section = '';
3834
- lines.forEach((line, index) => {
3835
- const header = line.trim().match(/^\[(project\.scripts|project\.entry-points\.[^\]]+)\]$/);
3836
- if (header) {
3837
- section = header[1];
3838
- return;
3839
- }
3840
- if (/^\[.+\]$/.test(line.trim()))
3841
- section = '';
3842
- if (!section)
3843
- return;
3844
- const entry = line.match(/^\s*([A-Za-z0-9_.-]+)\s*=\s*['"]([^'"]+)['"]/);
3845
- if (!entry)
3846
- return;
3847
- addRecord(records, {
3848
- kind: 'installed-entrypoint-state', projections: ['stateful_resources', 'registry_families', 'registry_entries'], presumptiveInventory: true,
3849
- path, line: index + 1, anchor: line, owner: section, identity: `${entry[1]}:${entry[2]}`,
3850
- tokens: [entry[1], entry[2]], selectorKey: entry[1], implementationKey: entry[2], addressToken: entry[1],
3851
- });
3852
- });
3853
- scanPythonBuildRequirements(path, lines, records);
3854
- }
3855
- if (lowerName === 'package.json') {
3856
- let parsed;
3857
- try {
3858
- parsed = JSON.parse(lines.join('\n'));
3859
- }
3860
- catch {
3861
- return;
3862
- }
3863
- const bins = typeof parsed.bin === 'string' ? { [basename(resolve(path), '.json')]: parsed.bin } : parsed.bin ?? {};
3864
- Object.entries(bins).forEach(([name, implementation]) => {
3865
- const lineIndex = lines.findIndex((line) => line.includes(JSON.stringify(name)) && line.includes(JSON.stringify(implementation)));
3866
- const line = lineIndex >= 0 ? lineIndex + 1 : 1;
3867
- addRecord(records, {
3868
- kind: 'installed-entrypoint-state', projections: ['stateful_resources', 'registry_families', 'registry_entries'], presumptiveInventory: true,
3869
- path, line, anchor: lines[line - 1] ?? '{', owner: 'package.bin',
3870
- identity: `${name}:${implementation}`, tokens: [name, implementation], selectorKey: name,
3871
- implementationKey: implementation, addressToken: name,
3872
- });
3873
- });
3874
- }
3875
- scanProductionManifestPackages(path, lines, records);
3876
- }
3877
- export function scanUniverseIdentities(repositoryRoot) {
3878
- const root = resolve(repositoryRoot);
3879
- const tracked = trackedFiles(root);
3880
- const requirementProfiles = discoverPythonRequirementProfiles(root, tracked);
3881
- const pythonLinesByPath = new Map();
3882
- const pythonCollectionsByPath = new Map();
3883
- const pythonCollectionsByModule = new Map();
3884
- for (const path of tracked.filter((candidate) => candidate.endsWith('.py') && shouldScan(candidate))) {
3885
- let lines = null;
3886
- try {
3887
- lines = textLines(root, path);
3888
- }
3889
- catch {
3890
- lines = null;
3891
- }
3892
- if (!lines)
3893
- continue;
3894
- pythonLinesByPath.set(path, lines);
3895
- const collections = parsePythonLiteralCollections(path, lines);
3896
- pythonCollectionsByPath.set(path, collections);
3897
- pythonCollectionsByModule.set(pythonModuleName(path), collections);
3898
- }
3899
- const pythonMatrixAnalyses = new Map();
3900
- const importedPythonCollectionIdentityKeys = new Set();
3901
- for (const [path, lines] of pythonLinesByPath) {
3902
- const bindings = bindPythonLiteralCollections(path, lines, pythonCollectionsByPath.get(path) ?? new Map(), pythonCollectionsByModule);
3903
- const ownIdentityKeys = new Set([...(pythonCollectionsByPath.get(path) ?? new Map()).values()]
3904
- .map((collection) => collection.symbolIdentityKey));
3905
- for (const collection of bindings.values()) {
3906
- if (!ownIdentityKeys.has(collection.symbolIdentityKey)) {
3907
- importedPythonCollectionIdentityKeys.add(collection.symbolIdentityKey);
3908
- }
3909
- }
3910
- pythonMatrixAnalyses.set(path, analyzePythonOperationMatrices(path, lines, bindings));
3911
- }
3912
- const localPythonModuleRoots = new Set();
3913
- for (const path of tracked.filter((candidate) => candidate.endsWith('.py'))) {
3914
- const parts = normalizePath(path).split('/');
3915
- if (parts[0] === 'src' && parts.length > 2)
3916
- localPythonModuleRoots.add(parts[1]);
3917
- else if (parts.length > 1)
3918
- localPythonModuleRoots.add(parts[0]);
3919
- else
3920
- localPythonModuleRoots.add(basename(parts[0], '.py'));
3921
- }
3922
- const records = new Map();
3923
- const omissions = [];
3924
- const manifestParts = [];
3925
- let scanned = 0;
3926
- for (const path of tracked) {
3927
- const absolute = join(root, path);
3928
- if (!existsSync(absolute) || !statSync(absolute).isFile())
3929
- continue;
3930
- if (!shouldScan(path) && !requirementProfiles.has(path))
3931
- continue;
3932
- const bytes = readFileSync(absolute);
3933
- manifestParts.push(`${path}\0${sha256(bytes)}\n`);
3934
- let lines;
3935
- try {
3936
- lines = pythonLinesByPath.get(path) ?? textLines(root, path);
3937
- }
3938
- catch {
3939
- lines = null;
3940
- }
3941
- if (!lines) {
3942
- omissions.push({ path, reason: 'binary-or-over-size-limit' });
3943
- continue;
3944
- }
3945
- scanned += 1;
3946
- const lowerName = basename(path).toLowerCase();
3947
- const extension = extname(lowerName);
3948
- const owners = extension === '.py' || extension === '.pyi'
3949
- ? pythonOwners(path, lines)
3950
- : lines.map(() => basename(path));
3951
- const executable = executableLineMask(path, lines);
3952
- scanEffects(path, lines, owners, executable, records);
3953
- scanExternalCallBlocks(path, lines, owners, executable, records);
3954
- scanJavaAndBrowserOperationalIdentities(path, lines, records);
3955
- scanWorkflowAndRecipeOperations(path, lines, records);
3956
- if (extension === '.py') {
3957
- scanPythonExecutableEntrypoint(path, lines, records);
3958
- scanPythonEmbeddedConfiguration(path, lines, pythonCollectionsByPath.get(path) ?? new Map(), importedPythonCollectionIdentityKeys, records);
3959
- scanPythonSelectors(path, lines, owners, executable, records);
3960
- scanPythonProtocolCallbacks(path, lines, owners, executable, records);
3961
- scanPythonOptionalImports(path, lines, owners, executable, records);
3962
- scanPythonImportedSdkOperations(path, lines, owners, executable, localPythonModuleRoots, records);
3963
- }
3964
- if (['.js', '.jsx', '.mjs', '.cjs', '.ts', '.tsx', '.mts', '.cts', '.java', '.kt', '.kts', '.cs', '.go', '.rs'].includes(extension)) {
3965
- scanGenericSelectors(path, lines, executable, records);
3966
- }
3967
- scanManifestEntrypoints(path, lines, records);
3968
- const requirementProfile = requirementProfiles.get(path);
3969
- if (requirementProfile)
3970
- scanPythonRequirementProfile(requirementProfile, lines, records);
3971
- if (lowerName === 'uv.lock')
3972
- scanUvLock(path, lines, records);
3973
- if (lowerName === 'package-lock.json' || lowerName === 'npm-shrinkwrap.json') {
3974
- scanPackageLock(path, lines.join('\n'), records);
3975
- }
3976
- }
3977
- for (const analysis of pythonMatrixAnalyses.values()) {
3978
- emitPythonExactOperationMatrices(analysis, records);
3979
- }
3980
- scanVueSelectorCensus(root, tracked, records);
3981
- scanJavaSelectorCensus(root, tracked, records);
3982
- scanJavaExactOperationMatrices(root, tracked, records);
3983
- scanPascalSelectorIdentities(root, tracked).forEach((record) => addRecord(records, record));
3984
- scanPythonManifestSelectorIdentities(root, tracked).forEach((record) => addRecord(records, record));
3985
- scanSpecializedOperationalIdentities(root, tracked).forEach((record) => addRecord(records, record));
3986
- const sortedRecords = [...records.values()].sort((left, right) => left.id.localeCompare(right.id));
3987
- const byKind = {};
3988
- const byProjection = {};
3989
- const byProofStatus = {
3990
- 'proof-complete': 0,
3991
- 'discovery-lead': 0,
3992
- 'non-inventory': 0,
3993
- };
3994
- sortedRecords.forEach((record) => {
3995
- byKind[record.kind] = (byKind[record.kind] ?? 0) + 1;
3996
- byProofStatus[record.proof_status] += 1;
3997
- record.projections.forEach((projection) => {
3998
- byProjection[projection] = (byProjection[projection] ?? 0) + 1;
3999
- });
4000
- });
4001
- // Hash incrementally so a large (but bounded per-record) census does not need
4002
- // a second repository-sized string allocation just to compute its receipt.
4003
- // Preserve the prior canonical encoding exactly: records are separated by a
4004
- // single newline, with no trailing newline.
4005
- const recordDigest = createHash('sha256');
4006
- sortedRecords.forEach((record, index) => {
4007
- if (index > 0)
4008
- recordDigest.update('\n');
4009
- recordDigest.update(JSON.stringify(record));
4010
- });
4011
- const projectionRecordIds = Object.fromEntries(['runtime_roles', 'stateful_resources', 'external_dependencies', 'registry_families', 'registry_entries', 'registry_edges']
4012
- .map((projection) => [
4013
- projection,
4014
- sortedRecords
4015
- .filter((record) => record.presumptive_inventory && record.projections.includes(projection))
4016
- .map((record) => record.id),
4017
- ]));
4018
- // Run the data-store pass before building the result so any inputs it cannot
4019
- // read are appended to `omissions` before `files_omitted` counts them.
4020
- const dataStores = collectDataStores(root, tracked, omissions);
4021
- return {
4022
- schema_version: IDENTITY_CENSUS_SCHEMA_VERSION,
4023
- contract_revision: IDENTITY_CENSUS_CONTRACT_REVISION,
4024
- repository_root: root,
4025
- source_commit: gitObjectId(root, 'HEAD^{commit}'),
4026
- source_tree: gitObjectId(root, 'HEAD^{tree}'),
4027
- source_manifest: {
4028
- digest: `sha256:${sha256(manifestParts.sort().join(''))}`,
4029
- tracked_files_total: tracked.length,
4030
- production_files_scanned: scanned,
4031
- files_omitted: omissions.length,
4032
- },
4033
- record_digest: `sha256:${recordDigest.digest('hex')}`,
4034
- counts: {
4035
- records: sortedRecords.length,
4036
- by_kind: Object.fromEntries(Object.entries(byKind).sort(([left], [right]) => left.localeCompare(right))),
4037
- by_projection: Object.fromEntries(Object.entries(byProjection).sort(([left], [right]) => left.localeCompare(right))),
4038
- by_proof_status: byProofStatus,
4039
- presumptive_inventory: sortedRecords.filter((record) => record.presumptive_inventory).length,
4040
- },
4041
- projection_record_ids: projectionRecordIds,
4042
- records: sortedRecords,
4043
- omissions,
4044
- data_stores: dataStores,
4045
- };
4046
- }
4047
- export function cloudVerifierIdentityCensusCommand(options) {
4048
- const root = options.root ? resolve(options.root) : (findGitRoot() ?? process.cwd());
4049
- const result = scanUniverseIdentities(root);
4050
- if (options.json) {
4051
- process.stdout.write(`${JSON.stringify(result, null, 2)}\n`);
4052
- return;
4053
- }
4054
- process.stdout.write(`Cloud Verifier identity census: ${result.counts.records} records from `
4055
- + `${result.source_manifest.production_files_scanned}/${result.source_manifest.tracked_files_total} files\n`);
4056
- process.stdout.write(`Record digest: ${result.record_digest}\n`);
4057
- Object.entries(result.counts.by_projection).forEach(([projection, count]) => {
4058
- process.stdout.write(` ${projection}: ${count}\n`);
4059
- });
4060
- }