@haystackeditor/cli 0.25.1 → 0.27.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +39 -463
- package/dist/capture/app-config.js +30 -6
- package/dist/commands/capture-brief.js +45 -35
- package/dist/commands/capture-contract.js +4 -4
- package/dist/commands/crawl-report.js +196 -0
- package/dist/commands/db-profile-upload.js +3 -3
- package/dist/commands/feedback.js +66 -0
- package/dist/commands/init-telemetry.js +55 -9
- package/dist/commands/init.js +8 -3
- package/dist/commands/lockfile-pin.js +307 -0
- package/dist/commands/telemetry-token.js +3 -3
- package/dist/commands/tokens.js +4 -4
- package/dist/commands/verify-explore.js +3 -3
- package/dist/commands/verify-history.js +2 -2
- package/dist/commands/verify-onboarding.js +13 -16
- package/dist/commands/verify-precompute.js +3 -4
- package/dist/commands/verify.js +31 -118
- package/dist/index.js +55 -968
- package/dist/schema.js +4 -10
- package/dist/utils/haystack-api.js +8 -36
- package/package.json +1 -5
- package/schemas/feedback.v1.json +13 -0
- package/schemas/pre-verify.v2.json +240 -0
- package/schemas/verify-raw.v1.json +1132 -0
- package/schemas/verify.v2.json +655 -0
- package/dist/assets/hooks/agent-context/detect.ts +0 -316
- package/dist/assets/hooks/agent-context/format.ts +0 -100
- package/dist/assets/hooks/agent-context/index.ts +0 -41
- package/dist/assets/hooks/agent-context/parsers/claude.ts +0 -262
- package/dist/assets/hooks/agent-context/parsers/codex.ts +0 -416
- package/dist/assets/hooks/agent-context/parsers/gemini.ts +0 -155
- package/dist/assets/hooks/agent-context/parsers/opencode.ts +0 -174
- package/dist/assets/hooks/agent-context/tsconfig.json +0 -14
- package/dist/assets/hooks/agent-context/types.ts +0 -58
- package/dist/assets/hooks/llm-rules-template.md +0 -59
- package/dist/assets/hooks/package-lock.json +0 -598
- package/dist/assets/hooks/package.json +0 -12
- package/dist/assets/hooks/scripts/commit-msg.sh +0 -5
- package/dist/assets/hooks/scripts/post-commit.sh +0 -5
- package/dist/assets/hooks/scripts/pre-commit.sh +0 -175
- package/dist/assets/hooks/scripts/pre-push.sh +0 -25
- package/dist/assets/hooks/scripts/prepare-commit-msg.sh +0 -5
- package/dist/assets/hooks/truncation-checker/ast-analyzer.ts +0 -528
- package/dist/assets/hooks/truncation-checker/index.ts +0 -595
- package/dist/assets/hooks/truncation-checker/tsconfig.json +0 -13
- package/dist/assets/skills/map-cloud-verifier-universe/SKILL.md +0 -2051
- package/dist/assets/skills/map-cloud-verifier-universe/agents/openai.yaml +0 -4
- package/dist/assets/skills/map-cloud-verifier-universe/references/output-contract.md +0 -3411
- package/dist/assets/skills/map-your-system.md +0 -143
- package/dist/assets/skills/submit.md +0 -200
- package/dist/commands/ask.js +0 -20
- package/dist/commands/cloud-verifier-behaviors.js +0 -218
- package/dist/commands/cloud-verifier-data-store-census.js +0 -539
- package/dist/commands/cloud-verifier-data-store-drift.js +0 -158
- package/dist/commands/cloud-verifier-identity-census.js +0 -4060
- package/dist/commands/cloud-verifier-materialization.js +0 -704
- package/dist/commands/cloud-verifier-pascal-selector-census.js +0 -1382
- package/dist/commands/cloud-verifier-python-manifest-selector-census.js +0 -2015
- package/dist/commands/cloud-verifier-specialized-operational-census.js +0 -11432
- package/dist/commands/cloud-verifier-universe.js +0 -10178
- package/dist/commands/config.js +0 -549
- package/dist/commands/design-verify.js +0 -311
- package/dist/commands/dismiss.js +0 -159
- package/dist/commands/hooks.js +0 -226
- package/dist/commands/inbox.js +0 -137
- package/dist/commands/mcp.js +0 -201
- package/dist/commands/policy.js +0 -371
- package/dist/commands/pr-status.js +0 -207
- package/dist/commands/pr.js +0 -105
- package/dist/commands/prepare-universe-review.js +0 -1092
- package/dist/commands/production-source-deny-policy.js +0 -100
- package/dist/commands/request-review.js +0 -74
- package/dist/commands/review.js +0 -191
- package/dist/commands/rules.js +0 -98
- package/dist/commands/scaffold-provisional-universe.js +0 -806
- package/dist/commands/setup.js +0 -1170
- package/dist/commands/skills.js +0 -447
- package/dist/commands/status.js +0 -35
- package/dist/commands/submit.js +0 -745
- package/dist/commands/system-map.js +0 -228
- package/dist/commands/triage.js +0 -598
- package/dist/commands/webhooks.js +0 -241
- package/dist/states.js +0 -46
- package/dist/tools/detect.js +0 -832
- package/dist/triage/astra.js +0 -202
- package/dist/triage/prompts.js +0 -188
- package/dist/triage/runner.js +0 -200
- package/dist/triage/types.js +0 -7
- package/dist/types.js +0 -326
- package/dist/utils/action-output.js +0 -26
- package/dist/utils/analysis-api.js +0 -416
- package/dist/utils/config.js +0 -54
- package/dist/utils/design-verifier-api.js +0 -294
- package/dist/utils/design-verifier-history.js +0 -79
- package/dist/utils/design-verifier-result.js +0 -424
- package/dist/utils/github-api.js +0 -324
- package/dist/utils/pending-state.js +0 -86
- package/dist/utils/pr-ref.js +0 -56
- package/dist/utils/prompter.js +0 -328
- package/schemas/action.v1.json +0 -22
- package/schemas/ask.v1.json +0 -40
- package/schemas/inbox.v1.json +0 -27
- package/schemas/pr-status.v1.json +0 -61
- package/schemas/pr.v1.json +0 -97
- package/schemas/pr.v3.json +0 -45
- package/schemas/setup.v1.json +0 -75
- package/schemas/submit.v1.json +0 -90
- package/schemas/triage.v1.json +0 -103
- package/schemas/triage.v2.json +0 -64
|
@@ -1,4060 +0,0 @@
|
|
|
1
|
-
import { createHash } from 'node:crypto';
|
|
2
|
-
import { execFileSync } from 'node:child_process';
|
|
3
|
-
import { existsSync, readFileSync, statSync } from 'node:fs';
|
|
4
|
-
import { basename, extname, join, posix, resolve } from 'node:path';
|
|
5
|
-
import fg from 'fast-glob';
|
|
6
|
-
import { findGitRoot } from '../utils/hooks.js';
|
|
7
|
-
import { scanPascalSelectorIdentities } from './cloud-verifier-pascal-selector-census.js';
|
|
8
|
-
import { scanPythonManifestSelectorIdentities } from './cloud-verifier-python-manifest-selector-census.js';
|
|
9
|
-
import { scanSpecializedOperationalIdentities } from './cloud-verifier-specialized-operational-census.js';
|
|
10
|
-
import { detectDataStores, } from './cloud-verifier-data-store-census.js';
|
|
11
|
-
export const IDENTITY_CENSUS_SCHEMA_VERSION = 'cloud-verifier-identity-census-v1';
|
|
12
|
-
export const IDENTITY_CENSUS_CONTRACT_REVISION = 39;
|
|
13
|
-
const MAX_SOURCE_BYTES = 4 * 1024 * 1024;
|
|
14
|
-
const MAX_LOCKFILE_BYTES = 24 * 1024 * 1024;
|
|
15
|
-
const MAX_EVIDENCE_TEXT_CHARS = 1_024;
|
|
16
|
-
const MAX_SUPPORTING_TOKENS = 64;
|
|
17
|
-
const LOCKFILE_NAMES = new Set([
|
|
18
|
-
'uv.lock',
|
|
19
|
-
'poetry.lock',
|
|
20
|
-
'pdm.lock',
|
|
21
|
-
'package-lock.json',
|
|
22
|
-
'npm-shrinkwrap.json',
|
|
23
|
-
'pnpm-lock.yaml',
|
|
24
|
-
'pnpm-lock.yml',
|
|
25
|
-
'yarn.lock',
|
|
26
|
-
'bun.lock',
|
|
27
|
-
'bun.lockb',
|
|
28
|
-
'cargo.lock',
|
|
29
|
-
'go.sum',
|
|
30
|
-
'gemfile.lock',
|
|
31
|
-
'composer.lock',
|
|
32
|
-
]);
|
|
33
|
-
const MANIFEST_NAMES = new Set([
|
|
34
|
-
'pyproject.toml',
|
|
35
|
-
'package.json',
|
|
36
|
-
'cargo.toml',
|
|
37
|
-
'go.mod',
|
|
38
|
-
'gemfile',
|
|
39
|
-
'composer.json',
|
|
40
|
-
'pom.xml',
|
|
41
|
-
'build.gradle',
|
|
42
|
-
'build.gradle.kts',
|
|
43
|
-
]);
|
|
44
|
-
const ALWAYS_SCAN_NAMES = new Set([
|
|
45
|
-
...LOCKFILE_NAMES,
|
|
46
|
-
...MANIFEST_NAMES,
|
|
47
|
-
'dockerfile',
|
|
48
|
-
'compose.yml',
|
|
49
|
-
'compose.yaml',
|
|
50
|
-
'docker-compose.yml',
|
|
51
|
-
'docker-compose.yaml',
|
|
52
|
-
'makefile',
|
|
53
|
-
'requirements.txt',
|
|
54
|
-
'requirements-prod.txt',
|
|
55
|
-
'requirements-production.txt',
|
|
56
|
-
'install.inf',
|
|
57
|
-
]);
|
|
58
|
-
const SOURCE_EXTENSIONS = new Set([
|
|
59
|
-
'.py', '.pyi', '.js', '.jsx', '.mjs', '.cjs', '.ts', '.tsx', '.mts', '.cts',
|
|
60
|
-
'.go', '.rs', '.rb', '.java', '.kt', '.kts', '.cs', '.php', '.swift', '.scala',
|
|
61
|
-
'.sh', '.bash', '.zsh', '.fish', '.ps1', '.sql', '.graphql', '.gql', '.proto',
|
|
62
|
-
'.toml', '.yaml', '.yml', '.json', '.xml', '.ini', '.cfg', '.conf', '.properties',
|
|
63
|
-
'.vue', '.pas', '.pp', '.inc', '.lpr', '.lpi', '.lpk', '.lfm', '.inf', '.plist',
|
|
64
|
-
'.desktop', '.control',
|
|
65
|
-
]);
|
|
66
|
-
const EXCLUDED_COMPONENTS = new Set([
|
|
67
|
-
'.git', '.hg', '.svn', '.haystack', '.agents', '.codex', '.claude',
|
|
68
|
-
'.venv', 'venv', 'node_modules', 'vendor', 'target',
|
|
69
|
-
'coverage', 'playwright-report', 'test-results', '__pycache__',
|
|
70
|
-
'test', 'tests', 'testapp', 'testapps', '__tests__', 'spec', 'specs', 'fixtures', '__fixtures__',
|
|
71
|
-
'examples', 'example', 'samples', 'sample', 'benchmarks', 'benchmark', 'docs', 'doc',
|
|
72
|
-
]);
|
|
73
|
-
// Repository modules often qualify a non-product purpose with the project name
|
|
74
|
-
// (for example, "project-docs" or "project-samples"). Match those purposes as
|
|
75
|
-
// exact structured component tokens, never as fuzzy substrings.
|
|
76
|
-
const EXCLUDED_DIRECTORY_PURPOSE_TOKENS = new Set([
|
|
77
|
-
'benchmark', 'benchmarks', 'doc', 'docs', 'documentation', 'example', 'examples',
|
|
78
|
-
'fixture', 'fixtures', 'sample', 'samples', 'spec', 'specs', 'test', 'tests',
|
|
79
|
-
]);
|
|
80
|
-
// Files the data-store detector reads: dependency manifests, service manifests,
|
|
81
|
-
// and config that name a database/cache/queue/store. A targeted, additive pass
|
|
82
|
-
// that does not alter the identity census's own scan surface.
|
|
83
|
-
const DATA_STORE_CONFIG_EXTENSIONS = new Set([
|
|
84
|
-
'.yml', '.yaml', '.toml', '.json', '.ini', '.conf', '.cfg', '.properties',
|
|
85
|
-
'.csproj', '.fsproj', '.vbproj', '.props',
|
|
86
|
-
]);
|
|
87
|
-
// Whether a file (by lowercased basename) is one the data-store detector reads.
|
|
88
|
-
// Exported so the drift check can gate on it: a change that touches no such file
|
|
89
|
-
// cannot alter the detected stores, so the drift check can skip it entirely.
|
|
90
|
-
export function isDataStoreInputName(lowerName) {
|
|
91
|
-
// A lockfile is the full transitive tree, not a store declaration, and can be
|
|
92
|
-
// large — reading it (as a .json/.yaml) only wastes work and inflates
|
|
93
|
-
// omissions. The detector reads the direct manifest (package.json) instead.
|
|
94
|
-
if (LOCKFILE_NAMES.has(lowerName))
|
|
95
|
-
return false;
|
|
96
|
-
return MANIFEST_NAMES.has(lowerName)
|
|
97
|
-
|| lowerName === 'mix.exs'
|
|
98
|
-
|| lowerName === 'packages.config'
|
|
99
|
-
// Django names its database/cache backends in settings.py, and Elixir names
|
|
100
|
-
// its stores in conventional config scripts (config/runtime.exs, etc.). These
|
|
101
|
-
// are specific, conventional basenames the config-declaration patterns target
|
|
102
|
-
// — fed by name so the whole `.py`/`.exs` source surface stays out.
|
|
103
|
-
|| lowerName === 'settings.py'
|
|
104
|
-
|| lowerName === 'config.exs' || lowerName === 'runtime.exs'
|
|
105
|
-
|| lowerName === 'prod.exs' || lowerName === 'dev.exs' || lowerName === 'releases.exs'
|
|
106
|
-
|| lowerName.startsWith('requirements')
|
|
107
|
-
|| lowerName === 'dockerfile' || lowerName.startsWith('dockerfile.') || lowerName.endsWith('.dockerfile')
|
|
108
|
-
|| lowerName === '.env' || lowerName.startsWith('.env.')
|
|
109
|
-
|| DATA_STORE_CONFIG_EXTENSIONS.has(extname(lowerName));
|
|
110
|
-
}
|
|
111
|
-
export function collectDataStores(root, tracked, omissions = []) {
|
|
112
|
-
const inputs = [];
|
|
113
|
-
for (const path of tracked) {
|
|
114
|
-
// Apply the census's own production-file boundary so this pass matches the
|
|
115
|
-
// identity records. This excludes the verifier's checked-in map
|
|
116
|
-
// (`.haystack/cloud-verifier/*.yml`, whose `image: postgres` evidence would
|
|
117
|
-
// otherwise mint stores from the map itself), plus test/example/vendor trees
|
|
118
|
-
// — a compose under `examples/` or `tests/` is not the app's own store. The
|
|
119
|
-
// drift check reuses this collector, so it inherits the same boundary.
|
|
120
|
-
if (isTestLike(path))
|
|
121
|
-
continue;
|
|
122
|
-
const lowerName = basename(path).toLowerCase();
|
|
123
|
-
if (!isDataStoreInputName(lowerName))
|
|
124
|
-
continue;
|
|
125
|
-
const absolute = join(root, path);
|
|
126
|
-
try {
|
|
127
|
-
if (!existsSync(absolute) || !statSync(absolute).isFile())
|
|
128
|
-
continue;
|
|
129
|
-
if (statSync(absolute).size > MAX_SOURCE_BYTES) {
|
|
130
|
-
omissions.push({ path, reason: 'data-store-input-over-size-limit' });
|
|
131
|
-
continue;
|
|
132
|
-
}
|
|
133
|
-
const bytes = readFileSync(absolute);
|
|
134
|
-
if (bytes.includes(0))
|
|
135
|
-
continue; // skip binary files
|
|
136
|
-
inputs.push({ path, lowerName, text: bytes.toString('utf8') });
|
|
137
|
-
}
|
|
138
|
-
catch {
|
|
139
|
-
// This pass reads files the main scan may not (.env, Dockerfiles,
|
|
140
|
-
// .csproj/.props, .ini), so its read failures are not covered by the
|
|
141
|
-
// main omission list — record them here so the gap stays visible.
|
|
142
|
-
omissions.push({ path, reason: 'data-store-input-unreadable' });
|
|
143
|
-
}
|
|
144
|
-
}
|
|
145
|
-
return detectDataStores(inputs);
|
|
146
|
-
}
|
|
147
|
-
function sha256(value) {
|
|
148
|
-
return createHash('sha256').update(value).digest('hex');
|
|
149
|
-
}
|
|
150
|
-
function stableId(kind, identityKey) {
|
|
151
|
-
const slug = kind.toLowerCase().replace(/[^a-z0-9]+/g, '-').replace(/^-|-$/g, '');
|
|
152
|
-
return `census-${slug}-${sha256(`${kind}\0${identityKey}`).slice(0, 20)}`;
|
|
153
|
-
}
|
|
154
|
-
function recordStableId(kind, path, owner, identity) {
|
|
155
|
-
return stableId(kind, `${path}#${owner}:${identity}`);
|
|
156
|
-
}
|
|
157
|
-
function normalizePath(path) {
|
|
158
|
-
return path.split(posix.sep).join('/').replace(/^\.\//, '');
|
|
159
|
-
}
|
|
160
|
-
function isTestLike(path) {
|
|
161
|
-
const sourceComponents = normalizePath(path).split('/');
|
|
162
|
-
const normalized = sourceComponents.join('/').toLowerCase();
|
|
163
|
-
const name = basename(normalized);
|
|
164
|
-
const components = normalized.split('/');
|
|
165
|
-
const productionSourceIndex = components.findIndex((component, index) => component === 'src' && components[index + 1] === 'main');
|
|
166
|
-
const insideProductionNamespace = (index) => productionSourceIndex >= 0 && index > productionSourceIndex + 1;
|
|
167
|
-
if (components.some((component, index) => !insideProductionNamespace(index) && EXCLUDED_COMPONENTS.has(component)))
|
|
168
|
-
return true;
|
|
169
|
-
const directoryHasExcludedPurpose = sourceComponents.slice(0, -1).some((component, index) => {
|
|
170
|
-
if (insideProductionNamespace(index))
|
|
171
|
-
return false;
|
|
172
|
-
const tokens = component
|
|
173
|
-
.replace(/([a-z0-9])([A-Z])/g, '$1-$2')
|
|
174
|
-
.toLowerCase()
|
|
175
|
-
.split(/[^a-z0-9]+/)
|
|
176
|
-
.filter(Boolean);
|
|
177
|
-
return tokens.some((token) => EXCLUDED_DIRECTORY_PURPOSE_TOKENS.has(token));
|
|
178
|
-
});
|
|
179
|
-
if (directoryHasExcludedPurpose)
|
|
180
|
-
return true;
|
|
181
|
-
if (ALWAYS_SCAN_NAMES.has(name))
|
|
182
|
-
return false;
|
|
183
|
-
return /(?:^|[._-])(test|tests|spec|fixture|mock)(?:[._-]|$)/.test(name)
|
|
184
|
-
|| /^(?:test_|conftest\.)/.test(name)
|
|
185
|
-
|| /(?:^|\.)stories\.[^.]+$/.test(name);
|
|
186
|
-
}
|
|
187
|
-
function shouldScan(path) {
|
|
188
|
-
const normalized = normalizePath(path);
|
|
189
|
-
const lowerName = basename(normalized).toLowerCase();
|
|
190
|
-
if (isTestLike(normalized))
|
|
191
|
-
return false;
|
|
192
|
-
if (ALWAYS_SCAN_NAMES.has(lowerName))
|
|
193
|
-
return true;
|
|
194
|
-
if (normalized.startsWith('.github/workflows/'))
|
|
195
|
-
return true;
|
|
196
|
-
if (normalized.startsWith('deploy/') || normalized.startsWith('deployment/') || normalized.startsWith('infra/'))
|
|
197
|
-
return true;
|
|
198
|
-
return SOURCE_EXTENSIONS.has(extname(lowerName));
|
|
199
|
-
}
|
|
200
|
-
function trackedFiles(root) {
|
|
201
|
-
try {
|
|
202
|
-
return execFileSync('git', ['-C', root, 'ls-files', '-z'], {
|
|
203
|
-
encoding: 'utf8',
|
|
204
|
-
maxBuffer: 64 * 1024 * 1024,
|
|
205
|
-
stdio: ['ignore', 'pipe', 'ignore'],
|
|
206
|
-
}).split('\0').filter(Boolean).map(normalizePath)
|
|
207
|
-
.filter((path) => !path.startsWith('.haystack/'))
|
|
208
|
-
.sort();
|
|
209
|
-
}
|
|
210
|
-
catch {
|
|
211
|
-
// Same entries as glob's { dot: true, nodir: true }: every non-directory,
|
|
212
|
-
// with symlinks listed as themselves and never followed.
|
|
213
|
-
return fg.sync('**/*', { cwd: root, dot: true, onlyFiles: false, followSymbolicLinks: false, markDirectories: true })
|
|
214
|
-
.filter((path) => !path.endsWith('/'))
|
|
215
|
-
.map(normalizePath)
|
|
216
|
-
.filter((path) => !path.startsWith('.git/') &&
|
|
217
|
-
!path.startsWith('.haystack/') &&
|
|
218
|
-
!path.includes('/node_modules/'))
|
|
219
|
-
.sort();
|
|
220
|
-
}
|
|
221
|
-
}
|
|
222
|
-
function gitObjectId(root, revision) {
|
|
223
|
-
try {
|
|
224
|
-
const value = execFileSync('git', ['-C', root, 'rev-parse', '--verify', revision], {
|
|
225
|
-
encoding: 'utf8',
|
|
226
|
-
maxBuffer: 1024 * 1024,
|
|
227
|
-
stdio: ['ignore', 'pipe', 'ignore'],
|
|
228
|
-
}).trim();
|
|
229
|
-
return /^[0-9a-f]{40}$/.test(value) ? value : null;
|
|
230
|
-
}
|
|
231
|
-
catch {
|
|
232
|
-
return null;
|
|
233
|
-
}
|
|
234
|
-
}
|
|
235
|
-
function textLines(root, path) {
|
|
236
|
-
const absolute = join(root, path);
|
|
237
|
-
const lowerName = basename(path).toLowerCase();
|
|
238
|
-
const maxBytes = LOCKFILE_NAMES.has(lowerName) ? MAX_LOCKFILE_BYTES : MAX_SOURCE_BYTES;
|
|
239
|
-
const stat = statSync(absolute);
|
|
240
|
-
if (!stat.isFile() || stat.size > maxBytes)
|
|
241
|
-
return null;
|
|
242
|
-
const bytes = readFileSync(absolute);
|
|
243
|
-
if (bytes.includes(0))
|
|
244
|
-
return null;
|
|
245
|
-
return bytes.toString('utf8').split(/\r?\n/);
|
|
246
|
-
}
|
|
247
|
-
// Compatibility for typed records introduced before proof_status became part
|
|
248
|
-
// of the census contract. New scanners must set proofStatus explicitly; this
|
|
249
|
-
// set only preserves the already-structured rev35/rev36 emitters.
|
|
250
|
-
const LEGACY_PROOF_COMPLETE_KINDS = new Set([
|
|
251
|
-
'locked-production-package-artifact-state',
|
|
252
|
-
'locked-production-package-artifact-install-operation',
|
|
253
|
-
'python-installation-profile-state',
|
|
254
|
-
'python-requirements-installation-profile-state',
|
|
255
|
-
'python-requirement-variant-state',
|
|
256
|
-
'python-requirements-profile-install-operation',
|
|
257
|
-
'exact-operation-matrix-member',
|
|
258
|
-
'maven-product-distribution-state',
|
|
259
|
-
'maven-installation-profile-state',
|
|
260
|
-
'maven-dependency-requirement-state',
|
|
261
|
-
'maven-plugin-dependency-requirement-state',
|
|
262
|
-
'maven-build-plugin-requirement-state',
|
|
263
|
-
'maven-plugin-execution-entrypoint-state',
|
|
264
|
-
'browser-keyed-storage-state',
|
|
265
|
-
'workflow-publication-stage-state',
|
|
266
|
-
'workflow-product-publication-runtime-unit',
|
|
267
|
-
'workflow-publication-secret-read-operation',
|
|
268
|
-
'workflow-publication-command-operation',
|
|
269
|
-
'workflow-publication-trigger-operation',
|
|
270
|
-
'workflow-publication-action-operation',
|
|
271
|
-
'workflow-publication-host-allocation-operation',
|
|
272
|
-
'browser-mounted-application-role',
|
|
273
|
-
'browser-supervised-subscription-role',
|
|
274
|
-
'spring-managed-lifecycle-role',
|
|
275
|
-
'java-annotated-lifecycle-role',
|
|
276
|
-
'java-supervised-worker-role',
|
|
277
|
-
'spring-static-resource-inbound-operation',
|
|
278
|
-
'spring-controller-inbound-operation',
|
|
279
|
-
'vue-local-component-family',
|
|
280
|
-
'vue-local-directive-family',
|
|
281
|
-
'vue-global-component-family',
|
|
282
|
-
'vue-global-directive-family',
|
|
283
|
-
'vue-selector-entry',
|
|
284
|
-
'vue-selector-edge',
|
|
285
|
-
'java-annotated-command-family',
|
|
286
|
-
'java-annotated-command-entry',
|
|
287
|
-
'java-json-subtype-family',
|
|
288
|
-
'java-json-subtype-entry',
|
|
289
|
-
'java-deserializer-family',
|
|
290
|
-
'java-deserializer-entry',
|
|
291
|
-
'java-spring-bean-alternative-family',
|
|
292
|
-
'java-spring-bean-alternative-entry',
|
|
293
|
-
'java-selector-edge',
|
|
294
|
-
]);
|
|
295
|
-
function addRecord(records, input) {
|
|
296
|
-
const identityKey = `${input.path}#${input.owner}:${input.identity}`;
|
|
297
|
-
const id = stableId(input.kind, identityKey);
|
|
298
|
-
if (records.has(id))
|
|
299
|
-
return;
|
|
300
|
-
const proofStatus = input.proofStatus ?? (LEGACY_PROOF_COMPLETE_KINDS.has(input.kind)
|
|
301
|
-
? 'proof-complete'
|
|
302
|
-
: input.presumptiveInventory ? 'discovery-lead' : 'non-inventory');
|
|
303
|
-
// Proof-complete records are mandatory finals. Operational discovery leads
|
|
304
|
-
// are also presumptive inventory: every exact source observation must be
|
|
305
|
-
// resolved to a mapped identity or an evidenced exclusion instead of
|
|
306
|
-
// disappearing behind an inventory-complete receipt. Registry-only leads
|
|
307
|
-
// remain in the selector-surface census until structured proof upgrades them.
|
|
308
|
-
const presumptiveInventory = proofStatus === 'proof-complete' || (proofStatus === 'discovery-lead' &&
|
|
309
|
-
input.projections.some((projection) => projection === 'runtime_roles' ||
|
|
310
|
-
projection === 'stateful_resources' ||
|
|
311
|
-
projection === 'external_dependencies'));
|
|
312
|
-
const rawIdentityComponents = {
|
|
313
|
-
...(input.identityComponents ?? {}),
|
|
314
|
-
};
|
|
315
|
-
if (proofStatus === 'proof-complete') {
|
|
316
|
-
rawIdentityComponents.identity_kind ??= input.kind;
|
|
317
|
-
rawIdentityComponents.production_provenance ??= `repository-source:${input.path}`;
|
|
318
|
-
rawIdentityComponents.owner_identity ??= `${input.path}#${input.owner}`;
|
|
319
|
-
if (input.projections.includes('registry_families')) {
|
|
320
|
-
rawIdentityComponents.selector_domain ??= input.identity;
|
|
321
|
-
}
|
|
322
|
-
if (input.projections.includes('registry_entries')) {
|
|
323
|
-
if (input.selectorKey !== undefined)
|
|
324
|
-
rawIdentityComponents.selector_key ??= input.selectorKey;
|
|
325
|
-
if (input.implementationKey !== undefined) {
|
|
326
|
-
rawIdentityComponents.implementation_identity ??= input.implementationKey;
|
|
327
|
-
}
|
|
328
|
-
}
|
|
329
|
-
if (input.projections.includes('registry_edges')) {
|
|
330
|
-
if (input.parentSelectorEntryRecordId !== undefined) {
|
|
331
|
-
rawIdentityComponents.parent_entry_identity ??= input.parentSelectorEntryRecordId;
|
|
332
|
-
}
|
|
333
|
-
if (input.childSelectorFamilyRecordId !== undefined) {
|
|
334
|
-
rawIdentityComponents.child_family_identity ??= input.childSelectorFamilyRecordId;
|
|
335
|
-
}
|
|
336
|
-
if (input.relationKind !== undefined)
|
|
337
|
-
rawIdentityComponents.relation_kind ??= input.relationKind;
|
|
338
|
-
}
|
|
339
|
-
if (input.projections.includes('runtime_roles')) {
|
|
340
|
-
rawIdentityComponents.controller_kind ??= input.kind.startsWith('workflow-')
|
|
341
|
-
? 'workflow'
|
|
342
|
-
: input.kind.startsWith('browser-') ? 'browser-host'
|
|
343
|
-
: input.kind.startsWith('spring-') ? 'spring-container'
|
|
344
|
-
: input.kind.startsWith('java-') ? 'jvm-application'
|
|
345
|
-
: 'source-declared';
|
|
346
|
-
rawIdentityComponents.entrypoint_identity ??= input.identity;
|
|
347
|
-
rawIdentityComponents.lifecycle_kind ??= input.operationToken ?? input.kind;
|
|
348
|
-
if (input.kind === 'workflow-product-publication-runtime-unit') {
|
|
349
|
-
rawIdentityComponents.production_operational_unit ??= 'true';
|
|
350
|
-
rawIdentityComponents.production_effect_kind ??= 'deploy';
|
|
351
|
-
}
|
|
352
|
-
}
|
|
353
|
-
if (input.projections.includes('stateful_resources')) {
|
|
354
|
-
rawIdentityComponents.address_grammar ??=
|
|
355
|
-
input.addressToken ?? input.materializationRootIdentityKey ??
|
|
356
|
-
input.packageArtifactUrl ?? input.packageArtifactFilename ?? input.identity;
|
|
357
|
-
rawIdentityComponents.reproduction_operation ??=
|
|
358
|
-
input.reproductionOperationToken ?? input.operationToken ?? `manifest-derived:${input.kind}`;
|
|
359
|
-
}
|
|
360
|
-
if (input.projections.includes('external_dependencies')) {
|
|
361
|
-
rawIdentityComponents.boundary_kind ??= input.kind;
|
|
362
|
-
rawIdentityComponents.direction ??= input.kind.includes('inbound') ? 'inbound' : 'outbound';
|
|
363
|
-
rawIdentityComponents.operation_identity ??=
|
|
364
|
-
input.operationIdentityKey ?? input.operationToken ?? input.identity;
|
|
365
|
-
rawIdentityComponents.target_identity ??=
|
|
366
|
-
input.addressToken ?? input.packageArtifactUrl ?? input.identity;
|
|
367
|
-
rawIdentityComponents.object_identity ??=
|
|
368
|
-
input.packageArtifactFilename ?? input.identity;
|
|
369
|
-
rawIdentityComponents.flow_stage ??= input.operationToken ?? input.kind;
|
|
370
|
-
}
|
|
371
|
-
}
|
|
372
|
-
const identityComponents = Object.keys(rawIdentityComponents).length === 0
|
|
373
|
-
? undefined
|
|
374
|
-
: Object.fromEntries(Object.entries(rawIdentityComponents)
|
|
375
|
-
.sort(([left], [right]) => left.localeCompare(right))
|
|
376
|
-
.map(([key, value]) => [key, Array.isArray(value) ? [...value] : value]));
|
|
377
|
-
const missingProofComponents = input.missingProofComponents ?? (proofStatus === 'proof-complete' ? [] : undefined);
|
|
378
|
-
records.set(id, {
|
|
379
|
-
id,
|
|
380
|
-
kind: input.kind,
|
|
381
|
-
projections: [...new Set(input.projections)].sort(),
|
|
382
|
-
presumptive_inventory: presumptiveInventory,
|
|
383
|
-
proof_status: proofStatus,
|
|
384
|
-
...(proofStatus !== 'proof-complete' ? {} : {
|
|
385
|
-
semantic_identity_id: stableId('semantic-identity-v1', input.semanticIdentityKey ?? identityKey),
|
|
386
|
-
}),
|
|
387
|
-
...(input.executionIdentityKey === undefined ? {} : {
|
|
388
|
-
execution_identity_id: stableId('execution-identity-v1', input.executionIdentityKey),
|
|
389
|
-
}),
|
|
390
|
-
...(identityComponents === undefined ? {} : { identity_components: identityComponents }),
|
|
391
|
-
...(missingProofComponents === undefined ? {} : {
|
|
392
|
-
missing_proof_components: [...new Set(missingProofComponents)].sort(),
|
|
393
|
-
}),
|
|
394
|
-
owner_identity_key: `${input.path}#${input.owner}`,
|
|
395
|
-
identity_key: identityKey,
|
|
396
|
-
...(input.selectorKey === undefined ? {} : { selector_key: input.selectorKey }),
|
|
397
|
-
...(input.implementationKey === undefined ? {} : { implementation_key: input.implementationKey }),
|
|
398
|
-
...(input.operationToken === undefined ? {} : { operation_token: input.operationToken }),
|
|
399
|
-
...(input.addressToken === undefined ? {} : { address_token: input.addressToken }),
|
|
400
|
-
...(input.packageName === undefined ? {} : { package_name: input.packageName }),
|
|
401
|
-
...(input.packageVersion === undefined ? {} : { package_version: input.packageVersion }),
|
|
402
|
-
...(input.packageArtifactKind === undefined ? {} : { package_artifact_kind: input.packageArtifactKind }),
|
|
403
|
-
...(input.packageArtifactFilename === undefined ? {} : { package_artifact_filename: input.packageArtifactFilename }),
|
|
404
|
-
...(input.packageArtifactUrl === undefined ? {} : { package_artifact_url: input.packageArtifactUrl }),
|
|
405
|
-
...(input.packageArtifactHash === undefined ? {} : { package_artifact_hash: input.packageArtifactHash }),
|
|
406
|
-
...(input.operationIdentityKey === undefined ? {} : { operation_identity_key: input.operationIdentityKey }),
|
|
407
|
-
...(input.materializationRootIdentityKey === undefined ? {} : {
|
|
408
|
-
materialization_root_identity_key: input.materializationRootIdentityKey,
|
|
409
|
-
}),
|
|
410
|
-
...(input.reproductionOperationToken === undefined ? {} : {
|
|
411
|
-
reproduction_operation_token: input.reproductionOperationToken,
|
|
412
|
-
}),
|
|
413
|
-
...(input.matrixAxes === undefined ? {} : {
|
|
414
|
-
matrix_axes: input.matrixAxes.map((axis) => ({
|
|
415
|
-
...axis,
|
|
416
|
-
source: {
|
|
417
|
-
...axis.source,
|
|
418
|
-
anchor: boundedEvidenceText(axis.source.anchor),
|
|
419
|
-
},
|
|
420
|
-
})),
|
|
421
|
-
}),
|
|
422
|
-
...(input.selectorFamilyRecordId === undefined ? {} : {
|
|
423
|
-
selector_family_record_id: input.selectorFamilyRecordId,
|
|
424
|
-
}),
|
|
425
|
-
...(input.parentSelectorEntryRecordId === undefined ? {} : {
|
|
426
|
-
parent_selector_entry_record_id: input.parentSelectorEntryRecordId,
|
|
427
|
-
}),
|
|
428
|
-
...(input.childSelectorFamilyRecordId === undefined ? {} : {
|
|
429
|
-
child_selector_family_record_id: input.childSelectorFamilyRecordId,
|
|
430
|
-
}),
|
|
431
|
-
...(input.relationKind === undefined ? {} : { relation_kind: input.relationKind }),
|
|
432
|
-
source: {
|
|
433
|
-
path: input.path,
|
|
434
|
-
start_line: input.line,
|
|
435
|
-
end_line: input.endLine ?? input.line,
|
|
436
|
-
anchor: boundedEvidenceText(input.anchor),
|
|
437
|
-
},
|
|
438
|
-
supporting_tokens: [...new Set(input.tokens.filter(Boolean))]
|
|
439
|
-
.slice(0, MAX_SUPPORTING_TOKENS)
|
|
440
|
-
.map(boundedEvidenceText),
|
|
441
|
-
});
|
|
442
|
-
}
|
|
443
|
-
function boundedEvidenceText(value) {
|
|
444
|
-
if (value.length <= MAX_EVIDENCE_TEXT_CHARS)
|
|
445
|
-
return value;
|
|
446
|
-
const digest = createHash('sha256').update(value).digest('hex');
|
|
447
|
-
const retained = Math.floor((MAX_EVIDENCE_TEXT_CHARS - digest.length - 32) / 2);
|
|
448
|
-
return `${value.slice(0, retained)} … [sha256:${digest}] … ${value.slice(-retained)}`;
|
|
449
|
-
}
|
|
450
|
-
function indentation(line) {
|
|
451
|
-
const prefix = line.match(/^\s*/)?.[0] ?? '';
|
|
452
|
-
return [...prefix].reduce((total, char) => total + (char === '\t' ? 8 : 1), 0);
|
|
453
|
-
}
|
|
454
|
-
function pythonOwners(path, lines) {
|
|
455
|
-
const owners = [];
|
|
456
|
-
const stack = [];
|
|
457
|
-
lines.forEach((line, index) => {
|
|
458
|
-
const trimmed = line.trim();
|
|
459
|
-
const indent = indentation(line);
|
|
460
|
-
const continuingSignature = stack.at(-1)?.signatureDepth ?? 0;
|
|
461
|
-
if (continuingSignature === 0 &&
|
|
462
|
-
trimmed &&
|
|
463
|
-
!trimmed.startsWith('#') &&
|
|
464
|
-
!trimmed.startsWith('@')) {
|
|
465
|
-
while (stack.length > 0 && indent <= stack[stack.length - 1].indent)
|
|
466
|
-
stack.pop();
|
|
467
|
-
}
|
|
468
|
-
const match = line.match(/^\s*(?:async\s+def|def|class)\s+([A-Za-z_][A-Za-z0-9_]*)/);
|
|
469
|
-
if (match)
|
|
470
|
-
stack.push({
|
|
471
|
-
indent,
|
|
472
|
-
name: match[1],
|
|
473
|
-
signatureDepth: Math.max(0, parenthesisDelta(line.slice(match.index ?? 0))),
|
|
474
|
-
});
|
|
475
|
-
else if (stack.length > 0 && stack[stack.length - 1].signatureDepth > 0) {
|
|
476
|
-
stack[stack.length - 1].signatureDepth = Math.max(0, stack[stack.length - 1].signatureDepth + parenthesisDelta(line));
|
|
477
|
-
}
|
|
478
|
-
owners[index] = stack.length > 0 ? stack.map((entry) => entry.name).join('.') : basename(path);
|
|
479
|
-
});
|
|
480
|
-
return owners;
|
|
481
|
-
}
|
|
482
|
-
function collectionBracketDelta(value) {
|
|
483
|
-
let delta = 0;
|
|
484
|
-
let quote = null;
|
|
485
|
-
let escaped = false;
|
|
486
|
-
for (const char of value) {
|
|
487
|
-
if (escaped) {
|
|
488
|
-
escaped = false;
|
|
489
|
-
continue;
|
|
490
|
-
}
|
|
491
|
-
if (char === '\\') {
|
|
492
|
-
escaped = true;
|
|
493
|
-
continue;
|
|
494
|
-
}
|
|
495
|
-
if (quote) {
|
|
496
|
-
if (char === quote)
|
|
497
|
-
quote = null;
|
|
498
|
-
continue;
|
|
499
|
-
}
|
|
500
|
-
if (char === "'" || char === '"') {
|
|
501
|
-
quote = char;
|
|
502
|
-
continue;
|
|
503
|
-
}
|
|
504
|
-
if (char === '[' || char === '(' || char === '{')
|
|
505
|
-
delta += 1;
|
|
506
|
-
if (char === ']' || char === ')' || char === '}')
|
|
507
|
-
delta -= 1;
|
|
508
|
-
}
|
|
509
|
-
return delta;
|
|
510
|
-
}
|
|
511
|
-
function pythonModuleName(path) {
|
|
512
|
-
const withoutExtension = normalizePath(path).replace(/\.pyi?$/, '');
|
|
513
|
-
const withoutSourceRoot = withoutExtension.startsWith('src/')
|
|
514
|
-
? withoutExtension.slice(4)
|
|
515
|
-
: withoutExtension;
|
|
516
|
-
return withoutSourceRoot.replace(/\/__init__$/, '').replaceAll('/', '.');
|
|
517
|
-
}
|
|
518
|
-
function stripPythonComment(value) {
|
|
519
|
-
let quote = null;
|
|
520
|
-
let escaped = false;
|
|
521
|
-
for (let index = 0; index < value.length; index += 1) {
|
|
522
|
-
const char = value[index];
|
|
523
|
-
if (escaped) {
|
|
524
|
-
escaped = false;
|
|
525
|
-
continue;
|
|
526
|
-
}
|
|
527
|
-
if (char === '\\') {
|
|
528
|
-
escaped = true;
|
|
529
|
-
continue;
|
|
530
|
-
}
|
|
531
|
-
if (quote) {
|
|
532
|
-
if (char === quote)
|
|
533
|
-
quote = null;
|
|
534
|
-
continue;
|
|
535
|
-
}
|
|
536
|
-
if (char === "'" || char === '"') {
|
|
537
|
-
quote = char;
|
|
538
|
-
continue;
|
|
539
|
-
}
|
|
540
|
-
if (char === '#')
|
|
541
|
-
return value.slice(0, index);
|
|
542
|
-
}
|
|
543
|
-
return value;
|
|
544
|
-
}
|
|
545
|
-
function parsePythonLiteralCollections(path, lines) {
|
|
546
|
-
const collections = new Map();
|
|
547
|
-
for (let index = 0; index < lines.length; index += 1) {
|
|
548
|
-
const assignment = lines[index].match(/^([A-Z][A-Z0-9_]*)\s*(?::[^=]+)?=\s*(\[|\()/);
|
|
549
|
-
if (!assignment)
|
|
550
|
-
continue;
|
|
551
|
-
const start = index;
|
|
552
|
-
let end = index;
|
|
553
|
-
let depth = collectionBracketDelta(lines[index].slice(lines[index].indexOf(assignment[2])));
|
|
554
|
-
while (depth > 0 && end + 1 < lines.length && end - start < 10_000) {
|
|
555
|
-
end += 1;
|
|
556
|
-
depth += collectionBracketDelta(lines[end]);
|
|
557
|
-
}
|
|
558
|
-
if (depth !== 0)
|
|
559
|
-
continue;
|
|
560
|
-
const symbol = assignment[1];
|
|
561
|
-
const symbolIdentityKey = `${path}#${symbol}`;
|
|
562
|
-
const items = [];
|
|
563
|
-
for (let itemLine = start; itemLine <= end; itemLine += 1) {
|
|
564
|
-
const text = stripPythonComment(itemLine === start
|
|
565
|
-
? lines[itemLine].slice(lines[itemLine].indexOf(assignment[2]) + 1)
|
|
566
|
-
: lines[itemLine]);
|
|
567
|
-
for (const match of text.matchAll(/(['"])(.*?)\1|\b-?\d+(?:\.\d+)?\b/g)) {
|
|
568
|
-
const value = match[2] ?? match[0];
|
|
569
|
-
const ordinal = items.length;
|
|
570
|
-
items.push({
|
|
571
|
-
value,
|
|
572
|
-
identityKey: `${symbolIdentityKey}[${ordinal}]`,
|
|
573
|
-
source: {
|
|
574
|
-
path,
|
|
575
|
-
start_line: itemLine + 1,
|
|
576
|
-
end_line: itemLine + 1,
|
|
577
|
-
anchor: lines[itemLine],
|
|
578
|
-
},
|
|
579
|
-
});
|
|
580
|
-
}
|
|
581
|
-
}
|
|
582
|
-
if (items.length === 0)
|
|
583
|
-
continue;
|
|
584
|
-
collections.set(symbol, {
|
|
585
|
-
symbol,
|
|
586
|
-
symbolIdentityKey,
|
|
587
|
-
source: {
|
|
588
|
-
path,
|
|
589
|
-
start_line: start + 1,
|
|
590
|
-
end_line: end + 1,
|
|
591
|
-
anchor: lines[start],
|
|
592
|
-
},
|
|
593
|
-
items,
|
|
594
|
-
});
|
|
595
|
-
index = end;
|
|
596
|
-
}
|
|
597
|
-
return collections;
|
|
598
|
-
}
|
|
599
|
-
function bindPythonLiteralCollections(path, lines, ownCollections, collectionsByModule) {
|
|
600
|
-
const bindings = new Map(ownCollections);
|
|
601
|
-
for (const line of lines) {
|
|
602
|
-
const fromImport = line.match(/^\s*from\s+([A-Za-z_][A-Za-z0-9_.]*)\s+import\s+(.+)$/);
|
|
603
|
-
if (fromImport) {
|
|
604
|
-
const imported = collectionsByModule.get(fromImport[1]);
|
|
605
|
-
if (!imported)
|
|
606
|
-
continue;
|
|
607
|
-
for (const rawSymbol of fromImport[2].replace(/[()]/g, '').split(',')) {
|
|
608
|
-
const symbol = rawSymbol.trim().match(/^([A-Za-z_][A-Za-z0-9_]*)(?:\s+as\s+([A-Za-z_][A-Za-z0-9_]*))?/);
|
|
609
|
-
if (!symbol)
|
|
610
|
-
continue;
|
|
611
|
-
const collection = imported.get(symbol[1]);
|
|
612
|
-
if (collection)
|
|
613
|
-
bindings.set(symbol[2] ?? symbol[1], collection);
|
|
614
|
-
}
|
|
615
|
-
continue;
|
|
616
|
-
}
|
|
617
|
-
const moduleImport = line.match(/^\s*import\s+([A-Za-z_][A-Za-z0-9_.]*)(?:\s+as\s+([A-Za-z_][A-Za-z0-9_]*))?/);
|
|
618
|
-
if (!moduleImport)
|
|
619
|
-
continue;
|
|
620
|
-
const imported = collectionsByModule.get(moduleImport[1]);
|
|
621
|
-
if (!imported)
|
|
622
|
-
continue;
|
|
623
|
-
const prefix = moduleImport[2] ?? moduleImport[1].split('.').at(-1);
|
|
624
|
-
for (const [symbol, collection] of imported) {
|
|
625
|
-
bindings.set(`${prefix}.${symbol}`, collection);
|
|
626
|
-
}
|
|
627
|
-
}
|
|
628
|
-
return bindings;
|
|
629
|
-
}
|
|
630
|
-
function pythonFunctionDefinitions(lines) {
|
|
631
|
-
const functions = new Map();
|
|
632
|
-
for (let index = 0; index < lines.length; index += 1) {
|
|
633
|
-
const match = lines[index].match(/^(\s*)(?:async\s+def|def)\s+([A-Za-z_][A-Za-z0-9_]*)\s*\(/);
|
|
634
|
-
if (!match)
|
|
635
|
-
continue;
|
|
636
|
-
const indent = indentation(match[1]);
|
|
637
|
-
let signatureEnd = index;
|
|
638
|
-
let signatureDepth = parenthesisDelta(lines[index].slice(lines[index].indexOf('(')));
|
|
639
|
-
while (signatureDepth > 0 && signatureEnd + 1 < lines.length) {
|
|
640
|
-
signatureEnd += 1;
|
|
641
|
-
signatureDepth += parenthesisDelta(lines[signatureEnd]);
|
|
642
|
-
}
|
|
643
|
-
let end = lines.length;
|
|
644
|
-
for (let cursor = signatureEnd + 1; cursor < lines.length; cursor += 1) {
|
|
645
|
-
const trimmed = lines[cursor].trim();
|
|
646
|
-
if (trimmed && !trimmed.startsWith('#') && indentation(lines[cursor]) <= indent) {
|
|
647
|
-
end = cursor;
|
|
648
|
-
break;
|
|
649
|
-
}
|
|
650
|
-
}
|
|
651
|
-
functions.set(match[2], {
|
|
652
|
-
name: match[2],
|
|
653
|
-
startLine: index + 1,
|
|
654
|
-
endLine: end,
|
|
655
|
-
indent,
|
|
656
|
-
});
|
|
657
|
-
}
|
|
658
|
-
return functions;
|
|
659
|
-
}
|
|
660
|
-
function mergePythonAxes(...groups) {
|
|
661
|
-
const byIdentity = new Map();
|
|
662
|
-
for (const group of groups) {
|
|
663
|
-
for (const collection of group)
|
|
664
|
-
byIdentity.set(collection.symbolIdentityKey, collection);
|
|
665
|
-
}
|
|
666
|
-
return [...byIdentity.values()].sort((left, right) => left.symbolIdentityKey.localeCompare(right.symbolIdentityKey));
|
|
667
|
-
}
|
|
668
|
-
function pythonAxisSetKey(axes) {
|
|
669
|
-
return axes.map((axis) => axis.symbolIdentityKey).sort().join('\0');
|
|
670
|
-
}
|
|
671
|
-
function analyzePythonOperationMatrices(path, lines, collectionBindings) {
|
|
672
|
-
const functions = pythonFunctionDefinitions(lines);
|
|
673
|
-
const loopsByFunction = new Map();
|
|
674
|
-
const callEdges = [];
|
|
675
|
-
const incoming = new Map([...functions.keys()].map((name) => [name, 0]));
|
|
676
|
-
for (const definition of functions.values()) {
|
|
677
|
-
const loops = [];
|
|
678
|
-
for (let lineIndex = definition.startLine; lineIndex < definition.endLine; lineIndex += 1) {
|
|
679
|
-
const line = lines[lineIndex];
|
|
680
|
-
const loop = line.match(/^\s*(?:async\s+)?for\s+.+?\s+in\s+(?:enumerate\(\s*)?([A-Za-z_][A-Za-z0-9_.]*)(?:\s*\))?\s*:/);
|
|
681
|
-
if (loop) {
|
|
682
|
-
const collection = collectionBindings.get(loop[1]);
|
|
683
|
-
if (collection) {
|
|
684
|
-
const loopIndent = indentation(line);
|
|
685
|
-
let loopEnd = definition.endLine;
|
|
686
|
-
for (let cursor = lineIndex + 1; cursor < definition.endLine; cursor += 1) {
|
|
687
|
-
const current = lines[cursor].trim();
|
|
688
|
-
if (current && !current.startsWith('#') && indentation(lines[cursor]) <= loopIndent) {
|
|
689
|
-
loopEnd = cursor;
|
|
690
|
-
break;
|
|
691
|
-
}
|
|
692
|
-
}
|
|
693
|
-
loops.push({ startLine: lineIndex + 1, endLine: loopEnd, collection });
|
|
694
|
-
}
|
|
695
|
-
}
|
|
696
|
-
for (const callee of functions.keys()) {
|
|
697
|
-
if (lineIndex + 1 === functions.get(callee)?.startLine)
|
|
698
|
-
continue;
|
|
699
|
-
const escaped = callee.replace(/[.*+?^${}()|[\]\\]/g, '\\$&');
|
|
700
|
-
if (!new RegExp(`\\b${escaped}\\s*\\(`).test(line))
|
|
701
|
-
continue;
|
|
702
|
-
const loopAxes = loops
|
|
703
|
-
.filter((candidate) => candidate.startLine <= lineIndex + 1 && lineIndex + 1 <= candidate.endLine)
|
|
704
|
-
.map((candidate) => candidate.collection);
|
|
705
|
-
const argumentAxes = [...collectionBindings.entries()]
|
|
706
|
-
.filter(([binding]) => {
|
|
707
|
-
const escapedBinding = binding.replace(/[.*+?^${}()|[\]\\]/g, '\\$&');
|
|
708
|
-
return new RegExp(`\\b${escapedBinding}\\b`).test(line);
|
|
709
|
-
})
|
|
710
|
-
.map(([, collection]) => collection);
|
|
711
|
-
callEdges.push({
|
|
712
|
-
caller: definition.name,
|
|
713
|
-
callee,
|
|
714
|
-
line: lineIndex + 1,
|
|
715
|
-
axes: mergePythonAxes(loopAxes, argumentAxes),
|
|
716
|
-
});
|
|
717
|
-
incoming.set(callee, (incoming.get(callee) ?? 0) + 1);
|
|
718
|
-
}
|
|
719
|
-
}
|
|
720
|
-
loopsByFunction.set(definition.name, loops);
|
|
721
|
-
}
|
|
722
|
-
const contextsByFunction = new Map();
|
|
723
|
-
for (const name of functions.keys()) {
|
|
724
|
-
contextsByFunction.set(name, incoming.get(name) === 0 ? [[]] : []);
|
|
725
|
-
}
|
|
726
|
-
let changed = true;
|
|
727
|
-
for (let pass = 0; changed && pass < functions.size * 4 + 4; pass += 1) {
|
|
728
|
-
changed = false;
|
|
729
|
-
for (const edge of callEdges) {
|
|
730
|
-
const callerContexts = contextsByFunction.get(edge.caller) ?? [];
|
|
731
|
-
const calleeContexts = contextsByFunction.get(edge.callee) ?? [];
|
|
732
|
-
const existing = new Set(calleeContexts.map(pythonAxisSetKey));
|
|
733
|
-
for (const callerContext of callerContexts) {
|
|
734
|
-
const context = mergePythonAxes(callerContext, edge.axes);
|
|
735
|
-
const key = pythonAxisSetKey(context);
|
|
736
|
-
if (existing.has(key))
|
|
737
|
-
continue;
|
|
738
|
-
calleeContexts.push(context);
|
|
739
|
-
existing.add(key);
|
|
740
|
-
changed = true;
|
|
741
|
-
}
|
|
742
|
-
contextsByFunction.set(edge.callee, calleeContexts);
|
|
743
|
-
}
|
|
744
|
-
}
|
|
745
|
-
return { path, lines, functions, loopsByFunction, contextsByFunction };
|
|
746
|
-
}
|
|
747
|
-
function scanPythonExecutableEntrypoint(path, lines, records) {
|
|
748
|
-
const functions = pythonFunctionDefinitions(lines);
|
|
749
|
-
for (let index = 0; index < lines.length; index += 1) {
|
|
750
|
-
const guard = lines[index].match(/^\s*if\s+(?:__name__\s*==\s*(['"])__main__\1|(['"])__main__\2\s*==\s*__name__)\s*:/);
|
|
751
|
-
if (!guard)
|
|
752
|
-
continue;
|
|
753
|
-
const guardIndent = indentation(lines[index]);
|
|
754
|
-
let end = lines.length;
|
|
755
|
-
for (let cursor = index + 1; cursor < lines.length; cursor += 1) {
|
|
756
|
-
const trimmed = lines[cursor].trim();
|
|
757
|
-
if (trimmed && !trimmed.startsWith('#') && indentation(lines[cursor]) <= guardIndent) {
|
|
758
|
-
end = cursor;
|
|
759
|
-
break;
|
|
760
|
-
}
|
|
761
|
-
}
|
|
762
|
-
const implementations = [...functions.keys()].filter((name) => {
|
|
763
|
-
const escaped = name.replace(/[.*+?^${}()|[\]\\]/g, '\\$&');
|
|
764
|
-
return lines.slice(index + 1, end).some((line) => new RegExp(`\\b${escaped}\\s*\\(`).test(line));
|
|
765
|
-
});
|
|
766
|
-
addRecord(records, {
|
|
767
|
-
kind: 'python-executable-entrypoint-role',
|
|
768
|
-
projections: ['runtime_roles'],
|
|
769
|
-
presumptiveInventory: true,
|
|
770
|
-
path,
|
|
771
|
-
line: index + 1,
|
|
772
|
-
anchor: lines[index],
|
|
773
|
-
owner: '__main__',
|
|
774
|
-
identity: `python-script:${path}:__main__`,
|
|
775
|
-
tokens: [path, '__main__', ...implementations],
|
|
776
|
-
implementationKey: implementations.join('|') || 'module-body',
|
|
777
|
-
operationToken: 'execute-python-script',
|
|
778
|
-
addressToken: path,
|
|
779
|
-
operationIdentityKey: `${path}#__main__`,
|
|
780
|
-
});
|
|
781
|
-
}
|
|
782
|
-
}
|
|
783
|
-
function scanPythonEmbeddedConfiguration(path, lines, collections, importedCollectionIdentityKeys, records) {
|
|
784
|
-
for (const collection of collections.values()) {
|
|
785
|
-
const used = importedCollectionIdentityKeys.has(collection.symbolIdentityKey) || lines.some((line, index) => index + 1 !== collection.source.start_line &&
|
|
786
|
-
new RegExp(`\\b${collection.symbol}\\b`).test(line));
|
|
787
|
-
if (!used)
|
|
788
|
-
continue;
|
|
789
|
-
addRecord(records, {
|
|
790
|
-
kind: 'python-embedded-configuration-state',
|
|
791
|
-
projections: ['stateful_resources'],
|
|
792
|
-
presumptiveInventory: true,
|
|
793
|
-
path,
|
|
794
|
-
line: collection.source.start_line,
|
|
795
|
-
anchor: collection.source.anchor,
|
|
796
|
-
owner: 'module-configuration',
|
|
797
|
-
identity: `embedded-configuration:${collection.symbolIdentityKey}`,
|
|
798
|
-
tokens: [collection.symbol, ...collection.items.map((item) => item.value)],
|
|
799
|
-
addressToken: collection.symbol,
|
|
800
|
-
materializationRootIdentityKey: collection.symbolIdentityKey,
|
|
801
|
-
});
|
|
802
|
-
}
|
|
803
|
-
lines.forEach((line, index) => {
|
|
804
|
-
const template = line.match(/^([A-Z][A-Z0-9_]*)\s*(?::[^=]+)?=\s*(?:[rubfRUBF]*)?(?:'''|""")/);
|
|
805
|
-
if (!template)
|
|
806
|
-
return;
|
|
807
|
-
const symbol = template[1];
|
|
808
|
-
const used = lines.some((candidate, candidateIndex) => candidateIndex !== index && new RegExp(`\\b${symbol}\\b`).test(candidate));
|
|
809
|
-
if (!used)
|
|
810
|
-
return;
|
|
811
|
-
addRecord(records, {
|
|
812
|
-
kind: 'python-embedded-template-state',
|
|
813
|
-
projections: ['stateful_resources'],
|
|
814
|
-
presumptiveInventory: true,
|
|
815
|
-
path,
|
|
816
|
-
line: index + 1,
|
|
817
|
-
anchor: line,
|
|
818
|
-
owner: 'module-configuration',
|
|
819
|
-
identity: `embedded-template:${path}#${symbol}`,
|
|
820
|
-
tokens: [symbol],
|
|
821
|
-
addressToken: symbol,
|
|
822
|
-
materializationRootIdentityKey: `${path}#${symbol}`,
|
|
823
|
-
});
|
|
824
|
-
});
|
|
825
|
-
}
|
|
826
|
-
const EXACT_MATRIX_OPERATION_KINDS = new Set([
|
|
827
|
-
'client-boundary-operation',
|
|
828
|
-
'http-operation',
|
|
829
|
-
'http-wrapper-operation',
|
|
830
|
-
'imported-sdk-instance-operation',
|
|
831
|
-
'socket-operation',
|
|
832
|
-
]);
|
|
833
|
-
function cartesianMatrixAxes(axes, limit = 25_000) {
|
|
834
|
-
const cardinality = axes.reduce((total, axis) => total * axis.items.length, 1);
|
|
835
|
-
if (!Number.isSafeInteger(cardinality) || cardinality > limit)
|
|
836
|
-
return null;
|
|
837
|
-
let rows = [[]];
|
|
838
|
-
for (const axis of axes) {
|
|
839
|
-
const next = [];
|
|
840
|
-
for (const row of rows) {
|
|
841
|
-
for (const item of axis.items) {
|
|
842
|
-
next.push([...row, {
|
|
843
|
-
symbol_identity_key: axis.symbolIdentityKey,
|
|
844
|
-
item_identity_key: item.identityKey,
|
|
845
|
-
item_value: item.value,
|
|
846
|
-
source: item.source,
|
|
847
|
-
}]);
|
|
848
|
-
}
|
|
849
|
-
}
|
|
850
|
-
rows = next;
|
|
851
|
-
}
|
|
852
|
-
return rows;
|
|
853
|
-
}
|
|
854
|
-
function emitPythonExactOperationMatrices(analysis, records) {
|
|
855
|
-
const groups = new Map();
|
|
856
|
-
for (const record of records.values()) {
|
|
857
|
-
if (record.source.path !== analysis.path ||
|
|
858
|
-
!EXACT_MATRIX_OPERATION_KINDS.has(record.kind) ||
|
|
859
|
-
!record.projections.includes('external_dependencies') ||
|
|
860
|
-
!record.operation_token)
|
|
861
|
-
continue;
|
|
862
|
-
const owner = record.owner_identity_key.slice(record.owner_identity_key.indexOf('#') + 1);
|
|
863
|
-
if (!analysis.functions.has(owner))
|
|
864
|
-
continue;
|
|
865
|
-
const key = `${owner}\0${record.source.start_line}\0${record.operation_token}`;
|
|
866
|
-
const group = groups.get(key) ?? [];
|
|
867
|
-
group.push(record);
|
|
868
|
-
groups.set(key, group);
|
|
869
|
-
}
|
|
870
|
-
for (const group of groups.values()) {
|
|
871
|
-
const representative = [...group].sort((left, right) => Number(right.presumptive_inventory) - Number(left.presumptive_inventory) ||
|
|
872
|
-
left.kind.localeCompare(right.kind))[0];
|
|
873
|
-
const owner = representative.owner_identity_key.slice(representative.owner_identity_key.indexOf('#') + 1);
|
|
874
|
-
const contexts = analysis.contextsByFunction.get(owner) ?? [];
|
|
875
|
-
const localAxes = (analysis.loopsByFunction.get(owner) ?? [])
|
|
876
|
-
.filter((loop) => loop.startLine <= representative.source.start_line &&
|
|
877
|
-
representative.source.start_line <= loop.endLine)
|
|
878
|
-
.map((loop) => loop.collection);
|
|
879
|
-
const axisSets = new Map();
|
|
880
|
-
let hasUnexpandedInvocation = false;
|
|
881
|
-
for (const context of contexts) {
|
|
882
|
-
const axes = mergePythonAxes(context, localAxes);
|
|
883
|
-
if (axes.length === 0) {
|
|
884
|
-
hasUnexpandedInvocation = true;
|
|
885
|
-
continue;
|
|
886
|
-
}
|
|
887
|
-
axisSets.set(pythonAxisSetKey(axes), axes);
|
|
888
|
-
}
|
|
889
|
-
if (axisSets.size === 0)
|
|
890
|
-
continue;
|
|
891
|
-
const operationToken = representative.operation_token;
|
|
892
|
-
const callsiteIdentity = `${analysis.path}#${owner}:${operationToken}` +
|
|
893
|
-
`@${representative.source.start_line}`;
|
|
894
|
-
for (const axes of axisSets.values()) {
|
|
895
|
-
const matrixRows = cartesianMatrixAxes(axes);
|
|
896
|
-
if (!matrixRows) {
|
|
897
|
-
hasUnexpandedInvocation = true;
|
|
898
|
-
continue;
|
|
899
|
-
}
|
|
900
|
-
for (const matrixAxes of matrixRows) {
|
|
901
|
-
const itemIdentity = matrixAxes.map((axis) => axis.item_identity_key).join('|');
|
|
902
|
-
const operationIdentityKey = `${callsiteIdentity}:${itemIdentity}`;
|
|
903
|
-
addRecord(records, {
|
|
904
|
-
kind: 'exact-operation-matrix-member',
|
|
905
|
-
projections: ['external_dependencies'],
|
|
906
|
-
presumptiveInventory: true,
|
|
907
|
-
path: analysis.path,
|
|
908
|
-
line: representative.source.start_line,
|
|
909
|
-
anchor: representative.source.anchor,
|
|
910
|
-
owner,
|
|
911
|
-
identity: operationIdentityKey,
|
|
912
|
-
tokens: [
|
|
913
|
-
operationToken,
|
|
914
|
-
...matrixAxes.map((axis) => axis.item_value),
|
|
915
|
-
],
|
|
916
|
-
operationToken,
|
|
917
|
-
addressToken: matrixAxes.map((axis) => axis.item_value).join('|'),
|
|
918
|
-
operationIdentityKey,
|
|
919
|
-
matrixAxes,
|
|
920
|
-
});
|
|
921
|
-
}
|
|
922
|
-
}
|
|
923
|
-
if (!hasUnexpandedInvocation) {
|
|
924
|
-
for (const record of group)
|
|
925
|
-
record.presumptive_inventory = false;
|
|
926
|
-
}
|
|
927
|
-
}
|
|
928
|
-
}
|
|
929
|
-
function executableLineMask(path, lines) {
|
|
930
|
-
const extension = extname(path.toLowerCase());
|
|
931
|
-
if (extension === '.pyi' || path.endsWith('.d.ts'))
|
|
932
|
-
return lines.map(() => false);
|
|
933
|
-
if (extension === '.vue') {
|
|
934
|
-
let inScript = false;
|
|
935
|
-
return lines.map((line) => {
|
|
936
|
-
if (/<script(?:\s[^>]*)?>/i.test(line)) {
|
|
937
|
-
inScript = true;
|
|
938
|
-
return false;
|
|
939
|
-
}
|
|
940
|
-
if (/<\/script>/i.test(line)) {
|
|
941
|
-
inScript = false;
|
|
942
|
-
return false;
|
|
943
|
-
}
|
|
944
|
-
const trimmed = line.trim();
|
|
945
|
-
return inScript && Boolean(trimmed) &&
|
|
946
|
-
!trimmed.startsWith('//') && !trimmed.startsWith('/*') && !trimmed.startsWith('*');
|
|
947
|
-
});
|
|
948
|
-
}
|
|
949
|
-
if (extension !== '.py') {
|
|
950
|
-
return lines.map((line) => {
|
|
951
|
-
const trimmed = line.trim();
|
|
952
|
-
return Boolean(trimmed) && !trimmed.startsWith('//') && !trimmed.startsWith('/*') && !trimmed.startsWith('*');
|
|
953
|
-
});
|
|
954
|
-
}
|
|
955
|
-
let delimiter = null;
|
|
956
|
-
return lines.map((line) => {
|
|
957
|
-
const trimmed = line.trim();
|
|
958
|
-
if (delimiter) {
|
|
959
|
-
const end = line.indexOf(delimiter);
|
|
960
|
-
if (end >= 0)
|
|
961
|
-
delimiter = null;
|
|
962
|
-
return false;
|
|
963
|
-
}
|
|
964
|
-
if (!trimmed || trimmed.startsWith('#'))
|
|
965
|
-
return false;
|
|
966
|
-
const single = line.indexOf("'''");
|
|
967
|
-
const double = line.indexOf('"""');
|
|
968
|
-
const start = single < 0 ? double : (double < 0 ? single : Math.min(single, double));
|
|
969
|
-
if (start < 0)
|
|
970
|
-
return true;
|
|
971
|
-
const selected = start === single ? "'''" : '"""';
|
|
972
|
-
if (line.indexOf(selected, start + 3) < 0)
|
|
973
|
-
delimiter = selected;
|
|
974
|
-
return start > 0 && Boolean(line.slice(0, start).trim());
|
|
975
|
-
});
|
|
976
|
-
}
|
|
977
|
-
function literalTokens(value) {
|
|
978
|
-
const tokens = [];
|
|
979
|
-
for (const match of value.matchAll(/(['"])(.*?)\1|\b(?:true|false|null|none)\b|\b-?\d+(?:\.\d+)?\b/gi)) {
|
|
980
|
-
const token = match[2] ?? match[0];
|
|
981
|
-
if (!tokens.includes(token))
|
|
982
|
-
tokens.push(token);
|
|
983
|
-
}
|
|
984
|
-
return tokens;
|
|
985
|
-
}
|
|
986
|
-
function selectorProjections(base, selectorSurface) {
|
|
987
|
-
const projections = [base];
|
|
988
|
-
if (/(?:^|[^a-z])(?:api|backend|callback|client|command|endpoint|event|export|format|grant|handler|hook|http|integration|listener|message|method|operation|provider|protocol|request|response|route|sdk|signal|stream|transport|url)(?:[^a-z]|$)/i
|
|
989
|
-
.test(selectorSurface)) {
|
|
990
|
-
projections.push('external_dependencies');
|
|
991
|
-
}
|
|
992
|
-
if (/(?:^|[^a-z])(?:cache|caches|callback|callbacks|config|configs|configuration|credential|credentials|hook|hooks|installed|patch|patches|plugin|plugins|queue|queues|registry|registries|session|sessions|state|states|store|stores|token|tokens|variable|variables|wrapper|wrappers)(?:[^a-z]|$)/i
|
|
993
|
-
.test(selectorSurface)) {
|
|
994
|
-
projections.push('stateful_resources');
|
|
995
|
-
}
|
|
996
|
-
return projections;
|
|
997
|
-
}
|
|
998
|
-
function scanPythonSelectors(path, lines, owners, executable, records) {
|
|
999
|
-
const activeChains = new Map();
|
|
1000
|
-
lines.forEach((line, index) => {
|
|
1001
|
-
if (!executable[index])
|
|
1002
|
-
return;
|
|
1003
|
-
const lineNumber = index + 1;
|
|
1004
|
-
const indent = indentation(line);
|
|
1005
|
-
const branch = line.match(/^\s*(if|elif)\s+(.+):\s*(?:#.*)?$/);
|
|
1006
|
-
if (branch) {
|
|
1007
|
-
const owner = owners[index];
|
|
1008
|
-
const prior = activeChains.get(indent);
|
|
1009
|
-
const chain = branch[1] === 'elif' && prior?.owner === owner
|
|
1010
|
-
? prior
|
|
1011
|
-
: { owner, line: lineNumber, condition: branch[2].trim() };
|
|
1012
|
-
activeChains.set(indent, chain);
|
|
1013
|
-
const selectors = literalTokens(branch[2]);
|
|
1014
|
-
const recognizedDispatch = selectors.length > 0
|
|
1015
|
-
|| /\b(?:isinstance|issubclass|len|type)\s*\(|\.(?:startswith|endswith)\s*\(/.test(branch[2]);
|
|
1016
|
-
if (!recognizedDispatch)
|
|
1017
|
-
return;
|
|
1018
|
-
const familyIdentity = `${chain.owner}:if-chain:${chain.line}`;
|
|
1019
|
-
const familyProjections = selectorProjections('registry_families', `${chain.owner} ${chain.condition}`);
|
|
1020
|
-
addRecord(records, {
|
|
1021
|
-
kind: 'conditional-dispatch-family', projections: familyProjections, presumptiveInventory: true,
|
|
1022
|
-
path, line: chain.line, anchor: lines[chain.line - 1], owner: chain.owner,
|
|
1023
|
-
identity: familyIdentity, tokens: [chain.condition], selectorKey: branch[2].trim(),
|
|
1024
|
-
});
|
|
1025
|
-
(selectors.length > 0 ? selectors : [branch[2].trim()]).forEach((selector) => {
|
|
1026
|
-
addRecord(records, {
|
|
1027
|
-
kind: 'conditional-dispatch-entry',
|
|
1028
|
-
projections: selectorProjections('registry_entries', `${chain.owner} ${branch[2]}`),
|
|
1029
|
-
presumptiveInventory: true,
|
|
1030
|
-
path, line: lineNumber, anchor: line, owner: chain.owner,
|
|
1031
|
-
identity: `${familyIdentity}:${selector}:${lineNumber}`, tokens: [branch[2].trim(), selector],
|
|
1032
|
-
selectorKey: selector, implementationKey: `branch@${path}:${lineNumber}`,
|
|
1033
|
-
});
|
|
1034
|
-
});
|
|
1035
|
-
}
|
|
1036
|
-
else if (line.trim() && !line.trim().startsWith('#')) {
|
|
1037
|
-
for (const chainIndent of [...activeChains.keys()]) {
|
|
1038
|
-
if (indent < chainIndent)
|
|
1039
|
-
activeChains.delete(chainIndent);
|
|
1040
|
-
}
|
|
1041
|
-
}
|
|
1042
|
-
const ternary = line.match(/(.+?)\s+if\s+(.+?)\s+else\s+(.+)/);
|
|
1043
|
-
if (ternary && !/^\s*(?:if|elif)\s/.test(line)) {
|
|
1044
|
-
const owner = owners[index];
|
|
1045
|
-
const familyIdentity = `${owner}:conditional-expression:${lineNumber}`;
|
|
1046
|
-
addRecord(records, {
|
|
1047
|
-
kind: 'conditional-expression-family',
|
|
1048
|
-
projections: selectorProjections('registry_families', `${owner} ${ternary[2]}`),
|
|
1049
|
-
presumptiveInventory: true,
|
|
1050
|
-
path, line: lineNumber, anchor: line, owner, identity: familyIdentity, tokens: [ternary[2].trim()],
|
|
1051
|
-
});
|
|
1052
|
-
[
|
|
1053
|
-
['true', ternary[1].trim()],
|
|
1054
|
-
['false', ternary[3].trim()],
|
|
1055
|
-
].forEach(([selector, implementation]) => addRecord(records, {
|
|
1056
|
-
kind: 'conditional-expression-entry',
|
|
1057
|
-
projections: selectorProjections('registry_entries', `${owner} ${ternary[2]}`),
|
|
1058
|
-
presumptiveInventory: true,
|
|
1059
|
-
path, line: lineNumber, anchor: line, owner,
|
|
1060
|
-
identity: `${familyIdentity}:${selector}`, tokens: [ternary[2].trim(), implementation],
|
|
1061
|
-
selectorKey: selector, implementationKey: implementation,
|
|
1062
|
-
}));
|
|
1063
|
-
}
|
|
1064
|
-
});
|
|
1065
|
-
for (let index = 0; index < lines.length; index += 1) {
|
|
1066
|
-
if (!executable[index])
|
|
1067
|
-
continue;
|
|
1068
|
-
const assignment = lines[index].match(/^\s*([A-Za-z_][A-Za-z0-9_.]*)\s*(?::[^=]+)?=\s*\{\s*$/);
|
|
1069
|
-
if (!assignment)
|
|
1070
|
-
continue;
|
|
1071
|
-
const owner = owners[index];
|
|
1072
|
-
const baseIndent = indentation(lines[index]);
|
|
1073
|
-
const entries = [];
|
|
1074
|
-
let end = index + 1;
|
|
1075
|
-
for (; end < lines.length; end += 1) {
|
|
1076
|
-
const current = lines[end];
|
|
1077
|
-
if (current.trim() === '}' && indentation(current) <= baseIndent)
|
|
1078
|
-
break;
|
|
1079
|
-
const entry = current.match(/^\s*(['"]?)([^'"\s][^:'"]*|[^'"]+)\1\s*:\s*(.+?)(?:,\s*)?(?:#.*)?$/);
|
|
1080
|
-
if (entry)
|
|
1081
|
-
entries.push({ line: end + 1, key: entry[2].trim(), value: entry[3].trim() });
|
|
1082
|
-
}
|
|
1083
|
-
if (entries.length < 2)
|
|
1084
|
-
continue;
|
|
1085
|
-
const familyIdentity = `${owner}:mapping:${assignment[1]}:${index + 1}`;
|
|
1086
|
-
addRecord(records, {
|
|
1087
|
-
kind: 'mapping-dispatch-family',
|
|
1088
|
-
projections: selectorProjections('registry_families', `${owner} ${assignment[1]}`),
|
|
1089
|
-
presumptiveInventory: true,
|
|
1090
|
-
path, line: index + 1, anchor: lines[index], owner, identity: familyIdentity, tokens: [assignment[1]],
|
|
1091
|
-
});
|
|
1092
|
-
entries.forEach((entry) => addRecord(records, {
|
|
1093
|
-
kind: 'mapping-dispatch-entry',
|
|
1094
|
-
projections: selectorProjections('registry_entries', `${owner} ${assignment[1]} ${entry.key}`),
|
|
1095
|
-
presumptiveInventory: true,
|
|
1096
|
-
path, line: entry.line, anchor: lines[entry.line - 1], owner,
|
|
1097
|
-
identity: `${familyIdentity}:${entry.key}`, tokens: [entry.key, entry.value],
|
|
1098
|
-
selectorKey: entry.key, implementationKey: entry.value,
|
|
1099
|
-
}));
|
|
1100
|
-
index = Math.max(index, end - 1);
|
|
1101
|
-
}
|
|
1102
|
-
}
|
|
1103
|
-
function scanGenericSelectors(path, lines, executable, records) {
|
|
1104
|
-
let switchOwner = null;
|
|
1105
|
-
lines.forEach((line, index) => {
|
|
1106
|
-
if (!executable[index])
|
|
1107
|
-
return;
|
|
1108
|
-
const lineNumber = index + 1;
|
|
1109
|
-
const switchMatch = line.match(/\bswitch\s*\((.+)\)/);
|
|
1110
|
-
if (switchMatch) {
|
|
1111
|
-
switchOwner = { line: lineNumber, expression: switchMatch[1].trim() };
|
|
1112
|
-
addRecord(records, {
|
|
1113
|
-
kind: 'switch-dispatch-family',
|
|
1114
|
-
projections: selectorProjections('registry_families', switchMatch[1]),
|
|
1115
|
-
presumptiveInventory: true,
|
|
1116
|
-
path, line: lineNumber, anchor: line, owner: `switch@${lineNumber}`,
|
|
1117
|
-
identity: `switch:${lineNumber}:${switchMatch[1].trim()}`, tokens: [switchMatch[1].trim()],
|
|
1118
|
-
});
|
|
1119
|
-
}
|
|
1120
|
-
const caseMatch = line.match(/\bcase\s+([^:]+):/);
|
|
1121
|
-
if (caseMatch && switchOwner) {
|
|
1122
|
-
const selector = caseMatch[1].trim().replace(/^['"]|['"]$/g, '');
|
|
1123
|
-
addRecord(records, {
|
|
1124
|
-
kind: 'switch-dispatch-entry',
|
|
1125
|
-
projections: selectorProjections('registry_entries', `${switchOwner.expression} ${caseMatch[1]}`),
|
|
1126
|
-
presumptiveInventory: true,
|
|
1127
|
-
path, line: lineNumber, anchor: line, owner: `switch@${switchOwner.line}`,
|
|
1128
|
-
identity: `switch:${switchOwner.line}:${selector}`, tokens: [caseMatch[1].trim()],
|
|
1129
|
-
selectorKey: selector, implementationKey: `branch@${path}:${lineNumber}`,
|
|
1130
|
-
});
|
|
1131
|
-
}
|
|
1132
|
-
const conditional = line.match(/\b(?:if|else\s+if)\s*\((.+)\)/);
|
|
1133
|
-
if (conditional) {
|
|
1134
|
-
const selectors = literalTokens(conditional[1]);
|
|
1135
|
-
if (selectors.length === 0 && !/\b(?:instanceof|typeof)\b|\.length\b|\.(?:startsWith|endsWith)\s*\(/.test(conditional[1]))
|
|
1136
|
-
return;
|
|
1137
|
-
const owner = `conditional@${lineNumber}`;
|
|
1138
|
-
addRecord(records, {
|
|
1139
|
-
kind: 'conditional-dispatch-family',
|
|
1140
|
-
projections: selectorProjections('registry_families', conditional[1]),
|
|
1141
|
-
presumptiveInventory: true,
|
|
1142
|
-
path, line: lineNumber, anchor: line, owner, identity: `${owner}:${conditional[1].trim()}`,
|
|
1143
|
-
tokens: [conditional[1].trim()],
|
|
1144
|
-
});
|
|
1145
|
-
(selectors.length > 0 ? selectors : [conditional[1].trim()]).forEach((selector) => addRecord(records, {
|
|
1146
|
-
kind: 'conditional-dispatch-entry',
|
|
1147
|
-
projections: selectorProjections('registry_entries', `${conditional[1]} ${selector}`),
|
|
1148
|
-
presumptiveInventory: true,
|
|
1149
|
-
path, line: lineNumber, anchor: line, owner, identity: `${owner}:${selector}`,
|
|
1150
|
-
tokens: [conditional[1].trim(), selector], selectorKey: selector,
|
|
1151
|
-
implementationKey: `branch@${path}:${lineNumber}`,
|
|
1152
|
-
}));
|
|
1153
|
-
}
|
|
1154
|
-
const ternaryIndex = line.indexOf('?');
|
|
1155
|
-
const colonIndex = ternaryIndex < 0 ? -1 : line.indexOf(':', ternaryIndex + 1);
|
|
1156
|
-
if (ternaryIndex > 0 && colonIndex > ternaryIndex) {
|
|
1157
|
-
const condition = line.slice(0, ternaryIndex).trim();
|
|
1158
|
-
const owner = `conditional-expression@${lineNumber}`;
|
|
1159
|
-
addRecord(records, {
|
|
1160
|
-
kind: 'conditional-expression-family',
|
|
1161
|
-
projections: selectorProjections('registry_families', condition),
|
|
1162
|
-
presumptiveInventory: true,
|
|
1163
|
-
path, line: lineNumber, anchor: line, owner, identity: `${owner}:${condition}`, tokens: [condition],
|
|
1164
|
-
});
|
|
1165
|
-
[['true', line.slice(ternaryIndex + 1, colonIndex).trim()], ['false', line.slice(colonIndex + 1).trim()]]
|
|
1166
|
-
.forEach(([selector, implementation]) => addRecord(records, {
|
|
1167
|
-
kind: 'conditional-expression-entry',
|
|
1168
|
-
projections: selectorProjections('registry_entries', `${condition} ${selector}`),
|
|
1169
|
-
presumptiveInventory: true,
|
|
1170
|
-
path, line: lineNumber, anchor: line, owner, identity: `${owner}:${selector}`,
|
|
1171
|
-
tokens: [condition, implementation], selectorKey: selector, implementationKey: implementation,
|
|
1172
|
-
}));
|
|
1173
|
-
}
|
|
1174
|
-
});
|
|
1175
|
-
}
|
|
1176
|
-
function resolvedSourceImport(importerPath, specifier, extension, aliases, knownPaths) {
|
|
1177
|
-
let joined = null;
|
|
1178
|
-
if (specifier.startsWith('.')) {
|
|
1179
|
-
joined = normalizePath(posix.normalize(posix.join(posix.dirname(importerPath), specifier)));
|
|
1180
|
-
}
|
|
1181
|
-
else {
|
|
1182
|
-
for (const [prefix, target] of aliases) {
|
|
1183
|
-
if (!specifier.startsWith(prefix))
|
|
1184
|
-
continue;
|
|
1185
|
-
joined = normalizePath(posix.normalize(posix.join(target, specifier.slice(prefix.length))));
|
|
1186
|
-
break;
|
|
1187
|
-
}
|
|
1188
|
-
}
|
|
1189
|
-
if (!joined)
|
|
1190
|
-
return null;
|
|
1191
|
-
const candidates = extname(joined)
|
|
1192
|
-
? [joined]
|
|
1193
|
-
: [`${joined}${extension}`, `${joined}/index${extension}`];
|
|
1194
|
-
return candidates.find((candidate) => knownPaths.has(candidate)) ?? candidates[0];
|
|
1195
|
-
}
|
|
1196
|
-
function scanVueSelectorCensus(root, paths, records) {
|
|
1197
|
-
const families = [];
|
|
1198
|
-
const knownPaths = new Set(paths.map(normalizePath));
|
|
1199
|
-
const aliases = new Map();
|
|
1200
|
-
for (const configPath of paths.filter((candidate) => /(?:^|\/)tsconfig(?:\.[^/]+)?\.json$/.test(candidate))) {
|
|
1201
|
-
let configLines = null;
|
|
1202
|
-
try {
|
|
1203
|
-
configLines = textLines(root, configPath);
|
|
1204
|
-
}
|
|
1205
|
-
catch {
|
|
1206
|
-
configLines = null;
|
|
1207
|
-
}
|
|
1208
|
-
if (!configLines)
|
|
1209
|
-
continue;
|
|
1210
|
-
const source = configLines.join('\n');
|
|
1211
|
-
for (const alias of source.matchAll(/"([^"]*\/)?\*"\s*:\s*\[\s*"([^"]*\/)?\*"/g)) {
|
|
1212
|
-
const prefix = alias[1] ?? '';
|
|
1213
|
-
const target = normalizePath(posix.join(posix.dirname(configPath), alias[2] ?? ''));
|
|
1214
|
-
if (prefix)
|
|
1215
|
-
aliases.set(prefix, target);
|
|
1216
|
-
}
|
|
1217
|
-
}
|
|
1218
|
-
for (const path of paths.filter((candidate) => candidate.endsWith('.vue') && shouldScan(candidate))) {
|
|
1219
|
-
let lines = null;
|
|
1220
|
-
try {
|
|
1221
|
-
lines = textLines(root, path);
|
|
1222
|
-
}
|
|
1223
|
-
catch {
|
|
1224
|
-
lines = null;
|
|
1225
|
-
}
|
|
1226
|
-
if (!lines)
|
|
1227
|
-
continue;
|
|
1228
|
-
const imports = new Map();
|
|
1229
|
-
lines.forEach((line) => {
|
|
1230
|
-
const imported = line.match(/^\s*import\s+([A-Za-z_$][\w$]*)\s+from\s+['"]([^'"]+)['"]/);
|
|
1231
|
-
if (!imported)
|
|
1232
|
-
return;
|
|
1233
|
-
const resolved = resolvedSourceImport(path, imported[2], '.vue', aliases, knownPaths);
|
|
1234
|
-
if (resolved)
|
|
1235
|
-
imports.set(imported[1], resolved);
|
|
1236
|
-
});
|
|
1237
|
-
for (let index = 0; index < lines.length; index += 1) {
|
|
1238
|
-
const opening = lines[index].match(/^\s*(components|directives)\s*:\s*\{\s*(.*?)\s*\}\s*,?\s*$/)
|
|
1239
|
-
?? lines[index].match(/^\s*(components|directives)\s*:\s*\{\s*$/);
|
|
1240
|
-
if (!opening)
|
|
1241
|
-
continue;
|
|
1242
|
-
const domain = opening[1];
|
|
1243
|
-
const entries = [];
|
|
1244
|
-
const sameLineBody = opening[2];
|
|
1245
|
-
if (sameLineBody) {
|
|
1246
|
-
sameLineBody.split(',').map((value) => value.trim()).filter(Boolean).forEach((value) => {
|
|
1247
|
-
const pair = value.match(/^(?:['"]([^'"]+)['"]|([A-Za-z_$][\w$-]*))\s*:\s*([A-Za-z_$][\w$]*)$/);
|
|
1248
|
-
const shorthand = value.match(/^([A-Za-z_$][\w$]*)$/);
|
|
1249
|
-
if (pair)
|
|
1250
|
-
entries.push({
|
|
1251
|
-
selector: pair[1] ?? pair[2], implementation: pair[3], line: index + 1, anchor: lines[index],
|
|
1252
|
-
});
|
|
1253
|
-
else if (shorthand)
|
|
1254
|
-
entries.push({
|
|
1255
|
-
selector: shorthand[1], implementation: shorthand[1], line: index + 1, anchor: lines[index],
|
|
1256
|
-
});
|
|
1257
|
-
});
|
|
1258
|
-
}
|
|
1259
|
-
else {
|
|
1260
|
-
const baseIndent = indentation(lines[index]);
|
|
1261
|
-
let cursor = index + 1;
|
|
1262
|
-
for (; cursor < lines.length; cursor += 1) {
|
|
1263
|
-
const current = lines[cursor];
|
|
1264
|
-
if (/^\s*}\s*,?\s*$/.test(current) && indentation(current) <= baseIndent)
|
|
1265
|
-
break;
|
|
1266
|
-
const pair = current.match(/^\s*(?:['"]([^'"]+)['"]|([A-Za-z_$][\w$-]*))\s*:\s*([A-Za-z_$][\w$]*)\s*,?\s*$/);
|
|
1267
|
-
const shorthand = current.match(/^\s*([A-Za-z_$][\w$]*)\s*,?\s*$/);
|
|
1268
|
-
if (pair)
|
|
1269
|
-
entries.push({
|
|
1270
|
-
selector: pair[1] ?? pair[2], implementation: pair[3], line: cursor + 1, anchor: current,
|
|
1271
|
-
});
|
|
1272
|
-
else if (shorthand)
|
|
1273
|
-
entries.push({
|
|
1274
|
-
selector: shorthand[1], implementation: shorthand[1], line: cursor + 1, anchor: current,
|
|
1275
|
-
});
|
|
1276
|
-
}
|
|
1277
|
-
index = cursor;
|
|
1278
|
-
}
|
|
1279
|
-
if (entries.length === 0)
|
|
1280
|
-
continue;
|
|
1281
|
-
families.push({
|
|
1282
|
-
path,
|
|
1283
|
-
owner: 'vue-component-options',
|
|
1284
|
-
identity: `vue-options:${domain}`,
|
|
1285
|
-
kind: domain === 'components' ? 'vue-local-component-family' : 'vue-local-directive-family',
|
|
1286
|
-
line: index + 1,
|
|
1287
|
-
anchor: lines[index] ?? `<${domain}>`,
|
|
1288
|
-
entries,
|
|
1289
|
-
imports,
|
|
1290
|
-
});
|
|
1291
|
-
}
|
|
1292
|
-
const globalEntries = new Map();
|
|
1293
|
-
lines.forEach((line, index) => {
|
|
1294
|
-
for (const match of line.matchAll(/\b([A-Za-z_$][\w$]*)\.(component|directive)\(\s*['"]([^'"]+)['"]\s*,\s*([A-Za-z_$][\w$]*)/g)) {
|
|
1295
|
-
const key = `${match[1]}.${match[2]}`;
|
|
1296
|
-
const entries = globalEntries.get(key) ?? [];
|
|
1297
|
-
entries.push({ selector: match[3], implementation: match[4], line: index + 1, anchor: line });
|
|
1298
|
-
globalEntries.set(key, entries);
|
|
1299
|
-
}
|
|
1300
|
-
});
|
|
1301
|
-
globalEntries.forEach((entries, key) => families.push({
|
|
1302
|
-
path,
|
|
1303
|
-
owner: `vue-registration:${key}`,
|
|
1304
|
-
identity: `vue-registration:${key}`,
|
|
1305
|
-
kind: key.endsWith('.component') ? 'vue-global-component-family' : 'vue-global-directive-family',
|
|
1306
|
-
line: entries[0].line,
|
|
1307
|
-
anchor: entries[0].anchor,
|
|
1308
|
-
entries,
|
|
1309
|
-
imports,
|
|
1310
|
-
}));
|
|
1311
|
-
}
|
|
1312
|
-
const familyIdByPath = new Map();
|
|
1313
|
-
const emittedEntries = [];
|
|
1314
|
-
families.forEach((family) => {
|
|
1315
|
-
const familyId = recordStableId(family.kind, family.path, family.owner, family.identity);
|
|
1316
|
-
const pathFamilies = familyIdByPath.get(family.path) ?? [];
|
|
1317
|
-
pathFamilies.push(familyId);
|
|
1318
|
-
familyIdByPath.set(family.path, pathFamilies);
|
|
1319
|
-
addRecord(records, {
|
|
1320
|
-
kind: family.kind,
|
|
1321
|
-
projections: ['registry_families'],
|
|
1322
|
-
presumptiveInventory: true,
|
|
1323
|
-
path: family.path,
|
|
1324
|
-
line: family.line,
|
|
1325
|
-
anchor: family.anchor,
|
|
1326
|
-
owner: family.owner,
|
|
1327
|
-
identity: family.identity,
|
|
1328
|
-
tokens: [family.identity],
|
|
1329
|
-
});
|
|
1330
|
-
family.entries.forEach((entry) => {
|
|
1331
|
-
const implementationPath = family.imports.get(entry.implementation);
|
|
1332
|
-
const implementationKey = implementationPath
|
|
1333
|
-
? `${implementationPath}#default-export`
|
|
1334
|
-
: `${family.path}#${entry.implementation}`;
|
|
1335
|
-
const entryIdentity = `${family.identity}:${entry.selector}:${implementationKey}`;
|
|
1336
|
-
const entryId = recordStableId('vue-selector-entry', family.path, family.owner, entryIdentity);
|
|
1337
|
-
addRecord(records, {
|
|
1338
|
-
kind: 'vue-selector-entry',
|
|
1339
|
-
projections: ['registry_entries'],
|
|
1340
|
-
presumptiveInventory: true,
|
|
1341
|
-
path: family.path,
|
|
1342
|
-
line: entry.line,
|
|
1343
|
-
anchor: entry.anchor,
|
|
1344
|
-
owner: family.owner,
|
|
1345
|
-
identity: entryIdentity,
|
|
1346
|
-
tokens: [entry.selector, entry.implementation, implementationKey],
|
|
1347
|
-
selectorKey: entry.selector,
|
|
1348
|
-
implementationKey,
|
|
1349
|
-
selectorFamilyRecordId: familyId,
|
|
1350
|
-
});
|
|
1351
|
-
emittedEntries.push({ entryId, family, implementation: entry.implementation });
|
|
1352
|
-
});
|
|
1353
|
-
});
|
|
1354
|
-
emittedEntries.forEach(({ entryId, family, implementation }) => {
|
|
1355
|
-
const implementationPath = family.imports.get(implementation);
|
|
1356
|
-
if (!implementationPath)
|
|
1357
|
-
return;
|
|
1358
|
-
const childFamilyIds = familyIdByPath.get(implementationPath) ?? [];
|
|
1359
|
-
childFamilyIds.forEach((childFamilyId) => addRecord(records, {
|
|
1360
|
-
kind: 'vue-selector-edge',
|
|
1361
|
-
projections: ['registry_edges'],
|
|
1362
|
-
presumptiveInventory: true,
|
|
1363
|
-
path: family.path,
|
|
1364
|
-
line: family.line,
|
|
1365
|
-
anchor: family.anchor,
|
|
1366
|
-
owner: family.owner,
|
|
1367
|
-
identity: `edge:${entryId}:${childFamilyId}`,
|
|
1368
|
-
tokens: [entryId, childFamilyId],
|
|
1369
|
-
parentSelectorEntryRecordId: entryId,
|
|
1370
|
-
childSelectorFamilyRecordId: childFamilyId,
|
|
1371
|
-
relationKind: 'implementation-owns-selector',
|
|
1372
|
-
}));
|
|
1373
|
-
});
|
|
1374
|
-
}
|
|
1375
|
-
function javaImportedTypes(lines) {
|
|
1376
|
-
const imports = new Map();
|
|
1377
|
-
lines.forEach((line) => {
|
|
1378
|
-
const imported = line.match(/^\s*import\s+(?:static\s+)?([A-Za-z_$][\w$]*(?:\.[A-Za-z_$][\w$]*)+)\s*;/)?.[1];
|
|
1379
|
-
if (imported && !imported.endsWith('.*'))
|
|
1380
|
-
imports.set(imported.split('.').at(-1), imported);
|
|
1381
|
-
});
|
|
1382
|
-
return imports;
|
|
1383
|
-
}
|
|
1384
|
-
function scanJavaSelectorCensus(root, paths, records) {
|
|
1385
|
-
const beanAlternatives = [];
|
|
1386
|
-
const commandFamilyIdsByClass = new Map();
|
|
1387
|
-
for (const path of paths.filter((candidate) => candidate.endsWith('.java') && shouldScan(candidate))) {
|
|
1388
|
-
let lines = null;
|
|
1389
|
-
try {
|
|
1390
|
-
lines = textLines(root, path);
|
|
1391
|
-
}
|
|
1392
|
-
catch {
|
|
1393
|
-
lines = null;
|
|
1394
|
-
}
|
|
1395
|
-
if (!lines)
|
|
1396
|
-
continue;
|
|
1397
|
-
const source = lines.join('\n');
|
|
1398
|
-
const imports = javaImportedTypes(lines);
|
|
1399
|
-
const className = [...source.matchAll(/\b(?:class|interface|record)\s+([A-Za-z_$][\w$]*)/g)].at(-1)?.[1]
|
|
1400
|
-
?? basename(path, '.java');
|
|
1401
|
-
const commandEntries = [];
|
|
1402
|
-
for (let index = 0; index < lines.length; index += 1) {
|
|
1403
|
-
if (!/@McpTool\s*\(/.test(lines[index]))
|
|
1404
|
-
continue;
|
|
1405
|
-
const annotation = [];
|
|
1406
|
-
let cursor = index;
|
|
1407
|
-
let depth = 0;
|
|
1408
|
-
do {
|
|
1409
|
-
annotation.push(lines[cursor]);
|
|
1410
|
-
depth += parenthesisDelta(lines[cursor]);
|
|
1411
|
-
cursor += 1;
|
|
1412
|
-
} while (cursor < lines.length && depth > 0 && cursor - index < 20);
|
|
1413
|
-
const annotationText = annotation.join('\n');
|
|
1414
|
-
const selector = annotationText.match(/\bname\s*=\s*"([^"]+)"/)?.[1];
|
|
1415
|
-
while (cursor < lines.length && (!lines[cursor].trim() || lines[cursor].trim().startsWith('@')))
|
|
1416
|
-
cursor += 1;
|
|
1417
|
-
const method = lines.slice(cursor, cursor + 6).join(' ').match(/\b(?:public|protected|private)?\s*(?:static\s+)?[A-Za-z_$][\w$]*(?:<[^;{}()]+>)?(?:\[\])?\s+([A-Za-z_$][\w$]*)\s*\(/)?.[1];
|
|
1418
|
-
if (selector && method)
|
|
1419
|
-
commandEntries.push({
|
|
1420
|
-
selector,
|
|
1421
|
-
implementation: `${path}#${className}.${method}`,
|
|
1422
|
-
line: index + 1,
|
|
1423
|
-
anchor: lines[index],
|
|
1424
|
-
});
|
|
1425
|
-
}
|
|
1426
|
-
if (commandEntries.length > 0) {
|
|
1427
|
-
const owner = `java-class:${className}`;
|
|
1428
|
-
const identity = `annotation-dispatch:McpTool.name:${className}`;
|
|
1429
|
-
const familyId = recordStableId('java-annotated-command-family', path, owner, identity);
|
|
1430
|
-
commandFamilyIdsByClass.set(className, familyId);
|
|
1431
|
-
addRecord(records, {
|
|
1432
|
-
kind: 'java-annotated-command-family',
|
|
1433
|
-
projections: ['registry_families'],
|
|
1434
|
-
presumptiveInventory: true,
|
|
1435
|
-
path,
|
|
1436
|
-
line: commandEntries[0].line,
|
|
1437
|
-
anchor: commandEntries[0].anchor,
|
|
1438
|
-
owner,
|
|
1439
|
-
identity,
|
|
1440
|
-
tokens: ['McpTool.name', className],
|
|
1441
|
-
});
|
|
1442
|
-
commandEntries.forEach((entry) => addRecord(records, {
|
|
1443
|
-
kind: 'java-annotated-command-entry',
|
|
1444
|
-
projections: ['registry_entries'],
|
|
1445
|
-
presumptiveInventory: true,
|
|
1446
|
-
path,
|
|
1447
|
-
line: entry.line,
|
|
1448
|
-
anchor: entry.anchor,
|
|
1449
|
-
owner,
|
|
1450
|
-
identity: `${identity}:${entry.selector}:${entry.implementation}`,
|
|
1451
|
-
tokens: [entry.selector, entry.implementation],
|
|
1452
|
-
selectorKey: entry.selector,
|
|
1453
|
-
implementationKey: entry.implementation,
|
|
1454
|
-
selectorFamilyRecordId: familyId,
|
|
1455
|
-
}));
|
|
1456
|
-
}
|
|
1457
|
-
for (const subtype of source.matchAll(/@JsonSubTypes\s*\(\s*\{([\s\S]*?)\}\s*\)\s*(?:public\s+)?(?:abstract\s+)?(?:class|interface)\s+([A-Za-z_$][\w$]*)/g)) {
|
|
1458
|
-
const ownerType = subtype[2];
|
|
1459
|
-
const entries = [...subtype[1].matchAll(/(?:JsonSubTypes\.)?Type\s*\(\s*value\s*=\s*([A-Za-z_$][\w$]*)\.class\s*,\s*name\s*=\s*([A-Za-z0-9_$."]+)/g)];
|
|
1460
|
-
if (entries.length === 0)
|
|
1461
|
-
continue;
|
|
1462
|
-
const line = source.slice(0, subtype.index ?? 0).split('\n').length;
|
|
1463
|
-
const owner = `java-type:${ownerType}`;
|
|
1464
|
-
const identity = `json-subtypes:${ownerType}`;
|
|
1465
|
-
const familyId = recordStableId('java-json-subtype-family', path, owner, identity);
|
|
1466
|
-
addRecord(records, {
|
|
1467
|
-
kind: 'java-json-subtype-family', projections: ['registry_families'], presumptiveInventory: true,
|
|
1468
|
-
path, line, anchor: lines[line - 1] ?? '@JsonSubTypes', owner, identity, tokens: [ownerType],
|
|
1469
|
-
});
|
|
1470
|
-
entries.forEach((entry) => {
|
|
1471
|
-
const implementation = imports.get(entry[1]) ?? entry[1];
|
|
1472
|
-
const selector = entry[2].replace(/^"|"$/g, '');
|
|
1473
|
-
addRecord(records, {
|
|
1474
|
-
kind: 'java-json-subtype-entry', projections: ['registry_entries'], presumptiveInventory: true,
|
|
1475
|
-
path, line, anchor: lines[line - 1] ?? '@JsonSubTypes', owner,
|
|
1476
|
-
identity: `${identity}:${selector}:${implementation}`,
|
|
1477
|
-
tokens: [selector, implementation], selectorKey: selector, implementationKey: implementation,
|
|
1478
|
-
selectorFamilyRecordId: familyId,
|
|
1479
|
-
});
|
|
1480
|
-
});
|
|
1481
|
-
}
|
|
1482
|
-
const deserializers = [...source.matchAll(/\baddDeserializer\s*\(\s*([A-Za-z_$][\w$]*)\.class\s*,\s*new\s+([A-Za-z_$][\w$]*)\s*\(/g)];
|
|
1483
|
-
if (deserializers.length > 0) {
|
|
1484
|
-
const owner = `java-class:${className}`;
|
|
1485
|
-
const identity = `jackson-deserializers:${className}`;
|
|
1486
|
-
const familyId = recordStableId('java-deserializer-family', path, owner, identity);
|
|
1487
|
-
const line = source.slice(0, deserializers[0].index ?? 0).split('\n').length;
|
|
1488
|
-
addRecord(records, {
|
|
1489
|
-
kind: 'java-deserializer-family', projections: ['registry_families'], presumptiveInventory: true,
|
|
1490
|
-
path, line, anchor: lines[line - 1] ?? 'addDeserializer', owner, identity, tokens: [className],
|
|
1491
|
-
});
|
|
1492
|
-
deserializers.forEach((entry) => {
|
|
1493
|
-
const entryLine = source.slice(0, entry.index ?? 0).split('\n').length;
|
|
1494
|
-
addRecord(records, {
|
|
1495
|
-
kind: 'java-deserializer-entry', projections: ['registry_entries'], presumptiveInventory: true,
|
|
1496
|
-
path, line: entryLine, anchor: lines[entryLine - 1] ?? entry[0], owner,
|
|
1497
|
-
identity: `${identity}:${entry[1]}:${entry[2]}`, tokens: [entry[1], entry[2]],
|
|
1498
|
-
selectorKey: imports.get(entry[1]) ?? entry[1], implementationKey: imports.get(entry[2]) ?? entry[2],
|
|
1499
|
-
selectorFamilyRecordId: familyId,
|
|
1500
|
-
});
|
|
1501
|
-
});
|
|
1502
|
-
}
|
|
1503
|
-
let pendingAnnotations = [];
|
|
1504
|
-
let annotationBuffer = null;
|
|
1505
|
-
lines.forEach((line, index) => {
|
|
1506
|
-
const trimmed = line.trim();
|
|
1507
|
-
if (annotationBuffer) {
|
|
1508
|
-
annotationBuffer.text += ` ${trimmed}`;
|
|
1509
|
-
annotationBuffer.depth += parenthesisDelta(line);
|
|
1510
|
-
if (annotationBuffer.depth <= 0) {
|
|
1511
|
-
pendingAnnotations.push({ line: annotationBuffer.line, text: annotationBuffer.text });
|
|
1512
|
-
annotationBuffer = null;
|
|
1513
|
-
}
|
|
1514
|
-
return;
|
|
1515
|
-
}
|
|
1516
|
-
if (trimmed.startsWith('@')) {
|
|
1517
|
-
const depth = parenthesisDelta(line);
|
|
1518
|
-
if (depth > 0)
|
|
1519
|
-
annotationBuffer = { line: index + 1, text: trimmed, depth };
|
|
1520
|
-
else
|
|
1521
|
-
pendingAnnotations.push({ line: index + 1, text: trimmed });
|
|
1522
|
-
return;
|
|
1523
|
-
}
|
|
1524
|
-
if (!trimmed || trimmed.startsWith('//'))
|
|
1525
|
-
return;
|
|
1526
|
-
if (!pendingAnnotations.some((annotation) => /^@Bean\b/.test(annotation.text))) {
|
|
1527
|
-
pendingAnnotations = [];
|
|
1528
|
-
return;
|
|
1529
|
-
}
|
|
1530
|
-
const method = line.match(/^\s*(?:public|protected|private)?\s*(?:static\s+)?([A-Za-z_$][\w$]*(?:<[^;{}()]+>)?(?:\[\])?)\s+([A-Za-z_$][\w$]*)\s*\(/);
|
|
1531
|
-
if (!method)
|
|
1532
|
-
return;
|
|
1533
|
-
const simpleContract = method[1].replace(/<.*$/, '');
|
|
1534
|
-
const contract = imports.get(simpleContract) ?? simpleContract;
|
|
1535
|
-
const conditions = pendingAnnotations
|
|
1536
|
-
.filter((annotation) => !/^@Bean\b/.test(annotation.text))
|
|
1537
|
-
.map((annotation) => annotation.text)
|
|
1538
|
-
.sort();
|
|
1539
|
-
beanAlternatives.push({
|
|
1540
|
-
path,
|
|
1541
|
-
line: index + 1,
|
|
1542
|
-
anchor: line,
|
|
1543
|
-
ownerClass: className,
|
|
1544
|
-
method: method[2],
|
|
1545
|
-
contract,
|
|
1546
|
-
selector: conditions.length > 0
|
|
1547
|
-
? conditions.join('&')
|
|
1548
|
-
: `unconditional:${path}#${className}.${method[2]}`,
|
|
1549
|
-
implementation: `${path}#${className}.${method[2]}`,
|
|
1550
|
-
});
|
|
1551
|
-
pendingAnnotations = [];
|
|
1552
|
-
});
|
|
1553
|
-
}
|
|
1554
|
-
const alternativesByContract = new Map();
|
|
1555
|
-
beanAlternatives.forEach((alternative) => {
|
|
1556
|
-
const group = alternativesByContract.get(alternative.contract) ?? [];
|
|
1557
|
-
group.push(alternative);
|
|
1558
|
-
alternativesByContract.set(alternative.contract, group);
|
|
1559
|
-
});
|
|
1560
|
-
alternativesByContract.forEach((alternatives, contract) => {
|
|
1561
|
-
const exactAlternatives = [...new Map(alternatives.map((entry) => [entry.implementation, entry])).values()]
|
|
1562
|
-
.sort((left, right) => left.implementation.localeCompare(right.implementation));
|
|
1563
|
-
if (exactAlternatives.length < 2 || exactAlternatives.every((entry) => entry.selector.startsWith('unconditional:')))
|
|
1564
|
-
return;
|
|
1565
|
-
const anchor = exactAlternatives[0];
|
|
1566
|
-
const owner = `spring-bean-contract:${contract}`;
|
|
1567
|
-
const identity = `spring-bean-alternatives:${contract}`;
|
|
1568
|
-
const familyId = recordStableId('java-spring-bean-alternative-family', anchor.path, owner, identity);
|
|
1569
|
-
addRecord(records, {
|
|
1570
|
-
kind: 'java-spring-bean-alternative-family', projections: ['registry_families'], presumptiveInventory: true,
|
|
1571
|
-
path: anchor.path, line: anchor.line, anchor: anchor.anchor, owner, identity, tokens: [contract],
|
|
1572
|
-
});
|
|
1573
|
-
exactAlternatives.forEach((entry) => {
|
|
1574
|
-
const entryIdentity = `${identity}:${entry.selector}:${entry.implementation}`;
|
|
1575
|
-
const entryId = recordStableId('java-spring-bean-alternative-entry', entry.path, owner, entryIdentity);
|
|
1576
|
-
addRecord(records, {
|
|
1577
|
-
kind: 'java-spring-bean-alternative-entry', projections: ['registry_entries'], presumptiveInventory: true,
|
|
1578
|
-
path: entry.path, line: entry.line, anchor: entry.anchor, owner, identity: entryIdentity,
|
|
1579
|
-
tokens: [contract, entry.selector, entry.implementation], selectorKey: entry.selector,
|
|
1580
|
-
implementationKey: entry.implementation, selectorFamilyRecordId: familyId,
|
|
1581
|
-
});
|
|
1582
|
-
const childFamilyId = commandFamilyIdsByClass.get(entry.contract.split('.').at(-1) ?? entry.contract);
|
|
1583
|
-
if (!childFamilyId)
|
|
1584
|
-
return;
|
|
1585
|
-
addRecord(records, {
|
|
1586
|
-
kind: 'java-selector-edge', projections: ['registry_edges'], presumptiveInventory: true,
|
|
1587
|
-
path: entry.path, line: entry.line, anchor: entry.anchor, owner,
|
|
1588
|
-
identity: `edge:${entryId}:${childFamilyId}`, tokens: [entryId, childFamilyId],
|
|
1589
|
-
parentSelectorEntryRecordId: entryId, childSelectorFamilyRecordId: childFamilyId,
|
|
1590
|
-
relationKind: 'implementation-owns-selector',
|
|
1591
|
-
});
|
|
1592
|
-
});
|
|
1593
|
-
});
|
|
1594
|
-
}
|
|
1595
|
-
function scanJavaExactOperationMatrices(root, paths, records) {
|
|
1596
|
-
const units = [];
|
|
1597
|
-
for (const path of paths.filter((candidate) => candidate.endsWith('.java') && shouldScan(candidate))) {
|
|
1598
|
-
let lines = null;
|
|
1599
|
-
try {
|
|
1600
|
-
lines = textLines(root, path);
|
|
1601
|
-
}
|
|
1602
|
-
catch {
|
|
1603
|
-
lines = null;
|
|
1604
|
-
}
|
|
1605
|
-
if (!lines)
|
|
1606
|
-
continue;
|
|
1607
|
-
const source = lines.join('\n');
|
|
1608
|
-
const className = source.match(/\b(?:class|record)\s+([A-Za-z_$][\w$]*)/)?.[1];
|
|
1609
|
-
if (className)
|
|
1610
|
-
units.push({ path, lines, source, className });
|
|
1611
|
-
}
|
|
1612
|
-
const unitsByClass = new Map(units.map((unit) => [unit.className, unit]));
|
|
1613
|
-
units.forEach((declarationUnit) => {
|
|
1614
|
-
for (const array of declarationUnit.source.matchAll(/\bString\s*\[\s*]\s*([A-Za-z_$][\w$]*)\s*=\s*\{([\s\S]*?)\}\s*;/g)) {
|
|
1615
|
-
const symbol = array[1];
|
|
1616
|
-
const values = [...array[2].matchAll(/"([^"]+)"/g)];
|
|
1617
|
-
if (values.length < 2)
|
|
1618
|
-
continue;
|
|
1619
|
-
const accessor = `get${symbol[0].toUpperCase()}${symbol.slice(1)}`;
|
|
1620
|
-
units.forEach((consumerUnit) => {
|
|
1621
|
-
const accessorUse = new RegExp(`\\.${accessor}\\s*\\(\\s*\\)`, 'g');
|
|
1622
|
-
for (const use of consumerUnit.source.matchAll(accessorUse)) {
|
|
1623
|
-
const useOffset = use.index ?? 0;
|
|
1624
|
-
const prefixStart = Math.max(0, useOffset - 1000);
|
|
1625
|
-
const constructors = [...consumerUnit.source.slice(prefixStart, useOffset)
|
|
1626
|
-
.matchAll(/new\s+([A-Za-z_$][\w$]*)\s*\(/g)];
|
|
1627
|
-
const constructor = constructors.at(-1);
|
|
1628
|
-
if (!constructor)
|
|
1629
|
-
continue;
|
|
1630
|
-
const implementation = unitsByClass.get(constructor[1]);
|
|
1631
|
-
if (!implementation)
|
|
1632
|
-
continue;
|
|
1633
|
-
const httpMethods = [...new Set([...implementation.source.matchAll(/\.(get|post|put|patch|delete|options|head)\s*\(\s*\)/gi)]
|
|
1634
|
-
.map((match) => match[1].toUpperCase()))];
|
|
1635
|
-
if (httpMethods.length !== 1 || !/\.uri\s*\(/.test(implementation.source))
|
|
1636
|
-
continue;
|
|
1637
|
-
const method = httpMethods[0];
|
|
1638
|
-
const arrayOffset = array.index ?? 0;
|
|
1639
|
-
const declarationLine = declarationUnit.source.slice(0, arrayOffset).split('\n').length;
|
|
1640
|
-
values.forEach((value, valueIndex) => {
|
|
1641
|
-
const itemValue = value[1];
|
|
1642
|
-
const itemOffset = arrayOffset + array[0].indexOf(array[2]) + (value.index ?? 0);
|
|
1643
|
-
const itemLine = declarationUnit.source.slice(0, itemOffset).split('\n').length;
|
|
1644
|
-
const configuredParts = itemValue.split(':');
|
|
1645
|
-
const endpointId = configuredParts[0];
|
|
1646
|
-
const endpointPath = configuredParts.length > 1 ? configuredParts.slice(1).join(':') : endpointId;
|
|
1647
|
-
const itemIdentityKey = `${declarationUnit.path}#${declarationUnit.className}.${symbol}[${valueIndex}]`;
|
|
1648
|
-
const operationIdentityKey = `${implementation.path}#${implementation.className}:${method}:matrix:${itemIdentityKey}`;
|
|
1649
|
-
addRecord(records, {
|
|
1650
|
-
kind: 'exact-operation-matrix-member',
|
|
1651
|
-
projections: ['external_dependencies'],
|
|
1652
|
-
presumptiveInventory: true,
|
|
1653
|
-
path: declarationUnit.path,
|
|
1654
|
-
line: itemLine,
|
|
1655
|
-
anchor: declarationUnit.lines[itemLine - 1] ?? declarationUnit.lines[declarationLine - 1] ?? array[0],
|
|
1656
|
-
owner: `${declarationUnit.className}.${symbol}`,
|
|
1657
|
-
identity: operationIdentityKey,
|
|
1658
|
-
tokens: [symbol, itemValue, endpointId, endpointPath, implementation.className, method],
|
|
1659
|
-
operationToken: method,
|
|
1660
|
-
addressToken: endpointPath,
|
|
1661
|
-
operationIdentityKey,
|
|
1662
|
-
matrixAxes: [{
|
|
1663
|
-
symbol_identity_key: `${declarationUnit.path}#${declarationUnit.className}.${symbol}`,
|
|
1664
|
-
item_identity_key: itemIdentityKey,
|
|
1665
|
-
item_value: itemValue,
|
|
1666
|
-
source: {
|
|
1667
|
-
path: declarationUnit.path,
|
|
1668
|
-
start_line: itemLine,
|
|
1669
|
-
end_line: itemLine,
|
|
1670
|
-
anchor: declarationUnit.lines[itemLine - 1] ?? itemValue,
|
|
1671
|
-
},
|
|
1672
|
-
}],
|
|
1673
|
-
});
|
|
1674
|
-
});
|
|
1675
|
-
}
|
|
1676
|
-
});
|
|
1677
|
-
}
|
|
1678
|
-
});
|
|
1679
|
-
}
|
|
1680
|
-
function enclosingTypeName(lines, index) {
|
|
1681
|
-
for (let cursor = index; cursor >= 0; cursor -= 1) {
|
|
1682
|
-
const type = lines[cursor].match(/\b(?:class|interface|record|enum)\s+([A-Za-z_$][\w$]*)/)?.[1];
|
|
1683
|
-
if (type)
|
|
1684
|
-
return type;
|
|
1685
|
-
}
|
|
1686
|
-
return basename('source');
|
|
1687
|
-
}
|
|
1688
|
-
function scanJavaAndBrowserOperationalIdentities(path, lines, records) {
|
|
1689
|
-
const extension = extname(path.toLowerCase());
|
|
1690
|
-
const javaLike = extension === '.java' || extension === '.kt' || extension === '.kts';
|
|
1691
|
-
const browserLike = ['.js', '.jsx', '.mjs', '.cjs', '.ts', '.tsx', '.mts', '.cts', '.vue']
|
|
1692
|
-
.includes(extension);
|
|
1693
|
-
if (browserLike) {
|
|
1694
|
-
lines.forEach((line, index) => {
|
|
1695
|
-
for (const mount of line.matchAll(/\b([A-Za-z_$][\w$]*)\.mount\(\s*([^,)]+)/g)) {
|
|
1696
|
-
const receiver = mount[1];
|
|
1697
|
-
const address = mount[2].trim();
|
|
1698
|
-
addRecord(records, {
|
|
1699
|
-
kind: 'browser-mounted-application-role',
|
|
1700
|
-
projections: ['runtime_roles'],
|
|
1701
|
-
presumptiveInventory: true,
|
|
1702
|
-
path,
|
|
1703
|
-
line: index + 1,
|
|
1704
|
-
anchor: line,
|
|
1705
|
-
owner: `browser-application:${receiver}`,
|
|
1706
|
-
identity: `mount:${receiver}:${address}`,
|
|
1707
|
-
tokens: [receiver, address],
|
|
1708
|
-
operationToken: 'mount',
|
|
1709
|
-
addressToken: address,
|
|
1710
|
-
operationIdentityKey: `${path}#${receiver}.mount:${address}`,
|
|
1711
|
-
});
|
|
1712
|
-
}
|
|
1713
|
-
for (const storage of line.matchAll(/\b(?:window\.)?(localStorage|sessionStorage)\.(getItem|setItem|removeItem)\(\s*['"]([^'"]+)['"]/g))
|
|
1714
|
-
addRecord(records, {
|
|
1715
|
-
kind: 'browser-keyed-storage-state',
|
|
1716
|
-
projections: ['stateful_resources'],
|
|
1717
|
-
presumptiveInventory: true,
|
|
1718
|
-
path,
|
|
1719
|
-
line: index + 1,
|
|
1720
|
-
anchor: line,
|
|
1721
|
-
owner: `browser-storage:${storage[1]}`,
|
|
1722
|
-
identity: `storage:${storage[1]}:${storage[3]}`,
|
|
1723
|
-
tokens: [storage[1], storage[2], storage[3]],
|
|
1724
|
-
addressToken: `${storage[1]}:${storage[3]}`,
|
|
1725
|
-
operationToken: storage[2],
|
|
1726
|
-
reproductionOperationToken: `${storage[1]}.${storage[2]}(${JSON.stringify(storage[3])})`,
|
|
1727
|
-
});
|
|
1728
|
-
for (const storage of line.matchAll(/\b(?:useStorage|useLocalStorage|useSessionStorage)\(\s*['"]([^'"]+)['"]/g))
|
|
1729
|
-
addRecord(records, {
|
|
1730
|
-
kind: 'browser-keyed-storage-state',
|
|
1731
|
-
projections: ['stateful_resources'],
|
|
1732
|
-
presumptiveInventory: true,
|
|
1733
|
-
path,
|
|
1734
|
-
line: index + 1,
|
|
1735
|
-
anchor: line,
|
|
1736
|
-
owner: 'browser-storage-composable',
|
|
1737
|
-
identity: `storage-composable:${storage[1]}`,
|
|
1738
|
-
tokens: [storage[0].split('(')[0], storage[1]],
|
|
1739
|
-
addressToken: storage[1],
|
|
1740
|
-
operationToken: storage[0].split('(')[0],
|
|
1741
|
-
reproductionOperationToken: storage[0].split('(')[0],
|
|
1742
|
-
});
|
|
1743
|
-
const browserRequests = [...line.matchAll(/\b(axios\.(get|post|put|patch|delete|options|head))\(\s*([`'"])(.*?)\3/g)].map((request) => ({
|
|
1744
|
-
callable: request[1],
|
|
1745
|
-
operation: request[2].toUpperCase(),
|
|
1746
|
-
address: request[4],
|
|
1747
|
-
offset: request.index ?? 0,
|
|
1748
|
-
anchor: line,
|
|
1749
|
-
endLine: index + 1,
|
|
1750
|
-
}));
|
|
1751
|
-
for (const fetchStart of line.matchAll(/\bfetch\s*\(/g)) {
|
|
1752
|
-
const offset = fetchStart.index ?? 0;
|
|
1753
|
-
const callLines = [line.slice(offset)];
|
|
1754
|
-
let depth = parenthesisDelta(callLines[0]);
|
|
1755
|
-
let cursor = index;
|
|
1756
|
-
while (depth > 0 && cursor + 1 < lines.length && cursor - index < 40) {
|
|
1757
|
-
cursor += 1;
|
|
1758
|
-
callLines.push(lines[cursor]);
|
|
1759
|
-
depth += parenthesisDelta(lines[cursor]);
|
|
1760
|
-
}
|
|
1761
|
-
const callAnchor = callLines.join('\n');
|
|
1762
|
-
const address = callAnchor.match(/\bfetch\s*\(\s*([`'"])(.*?)\1/s)?.[2];
|
|
1763
|
-
if (!address)
|
|
1764
|
-
continue;
|
|
1765
|
-
const declaredMethod = directFetchMethod(callAnchor);
|
|
1766
|
-
browserRequests.push({
|
|
1767
|
-
callable: 'fetch',
|
|
1768
|
-
operation: declaredMethod?.toUpperCase() ?? 'GET',
|
|
1769
|
-
address,
|
|
1770
|
-
offset,
|
|
1771
|
-
anchor: callAnchor,
|
|
1772
|
-
endLine: cursor + 1,
|
|
1773
|
-
});
|
|
1774
|
-
}
|
|
1775
|
-
for (const request of browserRequests) {
|
|
1776
|
-
addRecord(records, {
|
|
1777
|
-
kind: 'browser-http-request-operation',
|
|
1778
|
-
projections: ['external_dependencies'],
|
|
1779
|
-
presumptiveInventory: true,
|
|
1780
|
-
path,
|
|
1781
|
-
line: index + 1,
|
|
1782
|
-
endLine: request.endLine,
|
|
1783
|
-
anchor: request.anchor,
|
|
1784
|
-
owner: basename(path),
|
|
1785
|
-
identity: `browser-http:${request.operation}:${request.address}@${index + 1}:${request.offset}`,
|
|
1786
|
-
tokens: [request.callable, request.operation, request.address],
|
|
1787
|
-
operationToken: request.operation,
|
|
1788
|
-
addressToken: request.address,
|
|
1789
|
-
operationIdentityKey: `${path}#browser-http:${request.operation}:${request.address}@${index + 1}:${request.offset}`,
|
|
1790
|
-
});
|
|
1791
|
-
}
|
|
1792
|
-
});
|
|
1793
|
-
const source = lines.join('\n');
|
|
1794
|
-
for (const handle of source.matchAll(/(?:const|let|private|protected|public|readonly)\s+([A-Za-z_$][\w$]*)[^=;\n]*=\s*[^;\n]*\.subscribe\s*\(/g)) {
|
|
1795
|
-
const field = handle[1];
|
|
1796
|
-
if (!new RegExp(`\\b${field}\\.(?:unsubscribe|dispose|cancel)\\s*\\(`).test(source))
|
|
1797
|
-
continue;
|
|
1798
|
-
const line = source.slice(0, handle.index ?? 0).split('\n').length;
|
|
1799
|
-
addRecord(records, {
|
|
1800
|
-
kind: 'browser-supervised-subscription-role',
|
|
1801
|
-
projections: ['runtime_roles'],
|
|
1802
|
-
presumptiveInventory: true,
|
|
1803
|
-
path,
|
|
1804
|
-
line,
|
|
1805
|
-
anchor: lines[line - 1] ?? field,
|
|
1806
|
-
owner: basename(path),
|
|
1807
|
-
identity: `subscription:${field}`,
|
|
1808
|
-
tokens: [field, 'subscribe', 'unsubscribe'],
|
|
1809
|
-
operationToken: 'subscribe/unsubscribe',
|
|
1810
|
-
addressToken: field,
|
|
1811
|
-
operationIdentityKey: `${path}#subscription:${field}`,
|
|
1812
|
-
});
|
|
1813
|
-
}
|
|
1814
|
-
}
|
|
1815
|
-
if (!javaLike)
|
|
1816
|
-
return;
|
|
1817
|
-
const source = lines.join('\n');
|
|
1818
|
-
for (const bean of source.matchAll(/@Bean\s*\(\s*initMethod\s*=\s*"([^"]+)"\s*,\s*destroyMethod\s*=\s*"([^"]+)"\s*\)[\s\S]{0,500}?\b([A-Za-z_$][\w$<>?, ]*)\s+([A-Za-z_$][\w$]*)\s*\(/g)) {
|
|
1819
|
-
const line = source.slice(0, bean.index ?? 0).split('\n').length;
|
|
1820
|
-
addRecord(records, {
|
|
1821
|
-
kind: 'spring-managed-lifecycle-role',
|
|
1822
|
-
projections: ['runtime_roles'],
|
|
1823
|
-
presumptiveInventory: true,
|
|
1824
|
-
path,
|
|
1825
|
-
line,
|
|
1826
|
-
anchor: lines[line - 1] ?? '@Bean',
|
|
1827
|
-
owner: `spring-bean:${bean[4]}`,
|
|
1828
|
-
identity: `bean-lifecycle:${bean[4]}:${bean[1]}:${bean[2]}`,
|
|
1829
|
-
tokens: [bean[4], bean[1], bean[2], bean[3].trim()],
|
|
1830
|
-
operationToken: `${bean[1]}/${bean[2]}`,
|
|
1831
|
-
addressToken: bean[4],
|
|
1832
|
-
operationIdentityKey: `${path}#spring-bean:${bean[4]}:${bean[1]}:${bean[2]}`,
|
|
1833
|
-
});
|
|
1834
|
-
}
|
|
1835
|
-
lines.forEach((line, index) => {
|
|
1836
|
-
const staticResource = line.match(/\.addResourceHandler\s*\(([\s\S]*?)\)/);
|
|
1837
|
-
if (staticResource) {
|
|
1838
|
-
const route = [...staticResource[1].matchAll(/['"](\/[^'"]*)['"]/g)].at(-1)?.[1]
|
|
1839
|
-
?? staticResource[1].trim();
|
|
1840
|
-
for (const method of ['GET', 'HEAD'])
|
|
1841
|
-
addRecord(records, {
|
|
1842
|
-
kind: 'spring-static-resource-inbound-operation',
|
|
1843
|
-
projections: ['external_dependencies'],
|
|
1844
|
-
presumptiveInventory: true,
|
|
1845
|
-
path,
|
|
1846
|
-
line: index + 1,
|
|
1847
|
-
anchor: line,
|
|
1848
|
-
owner: enclosingTypeName(lines, index),
|
|
1849
|
-
identity: `static-resource:${method}:${route}@${index + 1}`,
|
|
1850
|
-
tokens: [method, route],
|
|
1851
|
-
operationToken: method,
|
|
1852
|
-
addressToken: route,
|
|
1853
|
-
operationIdentityKey: `${path}#static-resource:${method}:${route}@${index + 1}`,
|
|
1854
|
-
});
|
|
1855
|
-
}
|
|
1856
|
-
const mapping = line.match(/@(Get|Post|Put|Patch|Delete|Request)Mapping\s*\((.*)\)/);
|
|
1857
|
-
if (!mapping)
|
|
1858
|
-
return;
|
|
1859
|
-
const routes = [...mapping[2].matchAll(/['"](\/[^'"]*)['"]/g)].map((match) => match[1]);
|
|
1860
|
-
const methods = mapping[1] === 'Request'
|
|
1861
|
-
? [...mapping[2].matchAll(/RequestMethod\.(GET|POST|PUT|PATCH|DELETE|OPTIONS|HEAD)/g)].map((match) => match[1])
|
|
1862
|
-
: [mapping[1].toUpperCase()];
|
|
1863
|
-
(methods.length > 0 ? methods : ['ANY']).forEach((method) => (routes.length > 0 ? routes : ['class-mapping']).forEach((route) => addRecord(records, {
|
|
1864
|
-
kind: 'spring-controller-inbound-operation',
|
|
1865
|
-
projections: ['external_dependencies'],
|
|
1866
|
-
presumptiveInventory: true,
|
|
1867
|
-
path,
|
|
1868
|
-
line: index + 1,
|
|
1869
|
-
anchor: line,
|
|
1870
|
-
owner: enclosingTypeName(lines, index),
|
|
1871
|
-
identity: `controller:${method}:${route}@${index + 1}`,
|
|
1872
|
-
tokens: [method, route],
|
|
1873
|
-
operationToken: method,
|
|
1874
|
-
addressToken: route,
|
|
1875
|
-
operationIdentityKey: `${path}#controller:${method}:${route}@${index + 1}`,
|
|
1876
|
-
})));
|
|
1877
|
-
});
|
|
1878
|
-
const classBlocks = [...source.matchAll(/\bclass\s+([A-Za-z_$][\w$]*)/g)];
|
|
1879
|
-
classBlocks.forEach((classMatch, classIndex) => {
|
|
1880
|
-
const start = classMatch.index ?? 0;
|
|
1881
|
-
const end = classBlocks[classIndex + 1]?.index ?? source.length;
|
|
1882
|
-
const body = source.slice(start, end);
|
|
1883
|
-
if (!/@PostConstruct\b/.test(body) || !/@PreDestroy\b/.test(body))
|
|
1884
|
-
return;
|
|
1885
|
-
const line = source.slice(0, start).split('\n').length;
|
|
1886
|
-
addRecord(records, {
|
|
1887
|
-
kind: 'java-annotated-lifecycle-role',
|
|
1888
|
-
projections: ['runtime_roles'],
|
|
1889
|
-
presumptiveInventory: true,
|
|
1890
|
-
path,
|
|
1891
|
-
line,
|
|
1892
|
-
anchor: lines[line - 1] ?? classMatch[0],
|
|
1893
|
-
owner: classMatch[1],
|
|
1894
|
-
identity: `annotated-lifecycle:${classMatch[1]}`,
|
|
1895
|
-
tokens: [classMatch[1], 'PostConstruct', 'PreDestroy'],
|
|
1896
|
-
operationToken: 'PostConstruct/PreDestroy',
|
|
1897
|
-
addressToken: classMatch[1],
|
|
1898
|
-
operationIdentityKey: `${path}#class:${classMatch[1]}:PostConstruct/PreDestroy`,
|
|
1899
|
-
});
|
|
1900
|
-
});
|
|
1901
|
-
for (const field of source.matchAll(/\b(?:Disposable|Subscription|ScheduledFuture(?:<[^>]+>)?)\s+([A-Za-z_$][\w$]*)\s*[;=]/g)) {
|
|
1902
|
-
const fieldOffset = field.index ?? 0;
|
|
1903
|
-
let enclosingClassIndex = -1;
|
|
1904
|
-
for (let index = 0; index < classBlocks.length; index += 1) {
|
|
1905
|
-
if ((classBlocks[index].index ?? 0) >= fieldOffset)
|
|
1906
|
-
break;
|
|
1907
|
-
enclosingClassIndex = index;
|
|
1908
|
-
}
|
|
1909
|
-
if (enclosingClassIndex >= 0) {
|
|
1910
|
-
const classStart = classBlocks[enclosingClassIndex].index ?? 0;
|
|
1911
|
-
const classEnd = classBlocks[enclosingClassIndex + 1]?.index ?? source.length;
|
|
1912
|
-
const classBody = source.slice(classStart, classEnd);
|
|
1913
|
-
if (/@PostConstruct\b/.test(classBody) && /@PreDestroy\b/.test(classBody))
|
|
1914
|
-
continue;
|
|
1915
|
-
}
|
|
1916
|
-
const name = field[1];
|
|
1917
|
-
const startPattern = new RegExp(`\\b(?:this\\.)?${name}\\s*=\\s*[\\s\\S]{0,300}?(?:subscribe|schedule(?:AtFixedRate|WithFixedDelay)?)\\s*\\(`);
|
|
1918
|
-
const stopPattern = new RegExp(`\\b(?:this\\.)?${name}\\.(?:dispose|cancel|unsubscribe)\\s*\\(`);
|
|
1919
|
-
if (!startPattern.test(source) || !stopPattern.test(source))
|
|
1920
|
-
continue;
|
|
1921
|
-
const line = source.slice(0, field.index ?? 0).split('\n').length;
|
|
1922
|
-
const owner = enclosingTypeName(lines, line - 1);
|
|
1923
|
-
addRecord(records, {
|
|
1924
|
-
kind: 'java-supervised-worker-role',
|
|
1925
|
-
projections: ['runtime_roles'],
|
|
1926
|
-
presumptiveInventory: true,
|
|
1927
|
-
path,
|
|
1928
|
-
line,
|
|
1929
|
-
anchor: lines[line - 1] ?? field[0],
|
|
1930
|
-
owner,
|
|
1931
|
-
identity: `supervised-worker:${owner}:${name}`,
|
|
1932
|
-
tokens: [owner, name, field[0].trim()],
|
|
1933
|
-
operationToken: 'start/stop-handle',
|
|
1934
|
-
addressToken: name,
|
|
1935
|
-
operationIdentityKey: `${path}#${owner}:worker:${name}`,
|
|
1936
|
-
});
|
|
1937
|
-
}
|
|
1938
|
-
}
|
|
1939
|
-
const EFFECT_PATTERNS = [
|
|
1940
|
-
{ kind: 'filesystem-read-operation', projections: ['external_dependencies'], regex: /(?:\b(?:readFile(?:Sync)?|read_text|read_bytes|File\.read|fs\.readFile)|(?<!\.)\bopen)\s*\(/g },
|
|
1941
|
-
{ kind: 'filesystem-write-operation', projections: ['stateful_resources', 'external_dependencies'], regex: /\b(?:writeFile(?:Sync)?|appendFile(?:Sync)?|write_text|write_bytes|touch|mkdir|makedirs|unlink|rmdir|rmtree|File\.write|fs\.writeFile)\s*\(/g },
|
|
1942
|
-
{ kind: 'filesystem-metadata-operation', projections: ['external_dependencies'], regex: /\b(?:os\.(?:stat|lstat|listdir|scandir|walk|readlink|getcwd|chdir|chmod|rename|replace)|glob\.(?:glob|iglob)|shutil\.(?:copy|copy2|copyfile|copytree|move)|[A-Za-z0-9_.]+\.(?:exists|is_file|is_dir|stat|glob|rglob|iterdir|resolve|readlink|chmod|rename|replace|symlink_to|hardlink_to))\s*\(/g },
|
|
1943
|
-
{ kind: 'subprocess-operation', projections: ['external_dependencies'], regex: /\b(?:subprocess\.(?:run|Popen|call|check_call|check_output)|create_subprocess_(?:exec|shell)|child_process\.(?:spawn|exec|execFile|fork)|Command::new|ProcessBuilder|os\.(?:system|exec\w*))\s*\(/g },
|
|
1944
|
-
{ kind: 'command-wrapper-operation', projections: ['external_dependencies'], regex: /\b(?:run_command|run_cmd|execute_command|exec_command|invoke_command|shell_command)\s*\(\s*['"][^'"]+['"]/g },
|
|
1945
|
-
{ kind: 'host-execution-operation', projections: ['external_dependencies'], regex: /\b(?:runpy\.(?:run_module|run_path)|importlib\.(?:import_module|metadata\.(?:entry_points|distributions|version))|pkg_resources\.(?:iter_entry_points|get_distribution)|compile|exec)\s*\(/g },
|
|
1946
|
-
{ kind: 'host-installation-operation', projections: ['external_dependencies'], regex: /\b(?:entry_points|distributions|distribution|metadata|version)\s*\(|\.load\s*\(\s*\)/g },
|
|
1947
|
-
{ kind: 'host-interpreter-operation', projections: ['stateful_resources', 'external_dependencies'], regex: /\b(?:sys\.(?:meta_path|path_hooks|path_importer_cache|modules)|importlib\.(?:invalidate_caches|reload)|inspect\.(?:getsource|getattr_static|getmodule))\b/g },
|
|
1948
|
-
{ kind: 'socket-operation', projections: ['external_dependencies'], regex: /(?:\bsocket\.create_connection|\.(?:bind|listen|accept|connect|sendto|recvfrom))\s*\(/g },
|
|
1949
|
-
{ kind: 'host-signal-operation', projections: ['external_dependencies'], regex: /\b(?:signal\.signal|os\.kill|process\.(?:on|once)|[A-Za-z0-9_.]+\.add_signal_handler)\s*\(/g },
|
|
1950
|
-
{ kind: 'browser-host-operation', projections: ['external_dependencies'], regex: /\b(?:webbrowser\.open|window\.(?:open|addEventListener|removeEventListener)|document\.(?:createElement|createTextNode|getElementById|querySelector|querySelectorAll)|navigator\.[A-Za-z0-9_]+|Browser\.open|openExternal)\s*\(/g },
|
|
1951
|
-
{ kind: 'browser-model-operation', projections: ['external_dependencies'], regex: /\b(?:model|view)\.(?:addEventListener|addSignalListener|change|data|finalize|get|height|off|on|removeEventListener|removeSignalListener|resize|run|runAsync|save_changes|send|set|signal|toCanvas|toImageURL|toSVG|width)\s*\(/g },
|
|
1952
|
-
{ kind: 'browser-module-loader-operation', projections: ['external_dependencies'], regex: /\b(?:import|require|requirejs|define)(?:\.config)?\s*\(/g },
|
|
1953
|
-
{ kind: 'http-operation', projections: ['external_dependencies'], regex: /\b(?:fetch|requests\.(?:get|post|put|patch|delete|request)|axios\.(?:get|post|put|patch|delete|request)|httpx\.(?:get|post|put|patch|delete|request))\s*\(/g },
|
|
1954
|
-
{ kind: 'http-wrapper-operation', projections: ['external_dependencies'], regex: /\b(?:self\.)?_(?:get|post|put|patch|delete|request)(?:_raw)?\s*\(\s*(?:f|r|fr|rf)?['"]/g },
|
|
1955
|
-
{ kind: 'client-boundary-operation', projections: ['external_dependencies'], regex: /\b(?:[A-Za-z_][A-Za-z0-9_]*\.)*[A-Za-z0-9_]*(?:api|channel|client|connection|exporter|http|provider|session|transport)[A-Za-z0-9_]*\.(?:get|post|put|patch|delete|request|send|receive|respond|stream|query|execute|create|update|remove|list|load|export|publish|consume|ack|invoke|call)\s*\(/gi },
|
|
1956
|
-
{ kind: 'sdk-boundary-operation', projections: ['external_dependencies'], regex: /\b[A-Za-z_][A-Za-z0-9_.]*\.(?:add_event|add_link|attach|detach|emit|end|export|extract|force_flush|inject|invoke|observe|publish|record|receive_message|send_message|set_attribute|set_attributes|set_status|shutdown|start_span|write)\s*\(/g },
|
|
1957
|
-
{ kind: 'listener-operation', projections: ['external_dependencies'], regex: /\b(?:listen|serve|Server|Route|router\.(?:get|post|put|patch|delete)|app\.(?:get|post|put|patch|delete)|add_url_rule)\s*\(/g },
|
|
1958
|
-
{ kind: 'host-metadata-operation', projections: ['external_dependencies'], regex: /\b(?:socket\.gethostname|platform\.(?:machine|system|platform|node)|os\.(?:getpid|getppid|getuid|getgid|uname)|psutil\.(?:Process|cpu_percent|cpu_count|virtual_memory)|process\.(?:cpuUsage|memoryUsage))\s*\(/g },
|
|
1959
|
-
{ kind: 'installed-callback-state', projections: ['stateful_resources', 'external_dependencies'], regex: /\b(?:atexit\.register|os\.register_at_fork|signal\.signal|set_global_[A-Za-z0-9_]+|setattr|monkeypatch|patch)(?:\s*\(|\s*$)/g },
|
|
1960
|
-
{ kind: 'installed-registration-state', projections: ['stateful_resources'], regex: /\.(?:add_handler|add_hook|add_middleware|add_instrumentation|instrument)\s*\(/g },
|
|
1961
|
-
{ kind: 'in-memory-state', projections: ['stateful_resources'], regex: /\b(?:Queue|SimpleQueue|deque|WeakSet|WeakKeyDictionary|WeakValueDictionary|LRUCache|TTLCache)\s*\(/g },
|
|
1962
|
-
{ kind: 'runtime-task-candidate', projections: ['runtime_roles'], regex: /\b(?:asyncio\.create_task|create_task|ensure_future|Thread|Process|ThreadPoolExecutor|ProcessPoolExecutor)\s*\(/g, presumptive: false },
|
|
1963
|
-
];
|
|
1964
|
-
function scanEffects(path, lines, owners, executable, records) {
|
|
1965
|
-
lines.forEach((line, index) => {
|
|
1966
|
-
if (!executable[index])
|
|
1967
|
-
return;
|
|
1968
|
-
const owner = owners[index] ?? basename(path);
|
|
1969
|
-
for (const pattern of EFFECT_PATTERNS) {
|
|
1970
|
-
pattern.regex.lastIndex = 0;
|
|
1971
|
-
let match;
|
|
1972
|
-
while ((match = pattern.regex.exec(line)) !== null) {
|
|
1973
|
-
const token = match[0].replace(/\s*\($/, '').trim();
|
|
1974
|
-
addRecord(records, {
|
|
1975
|
-
kind: pattern.kind,
|
|
1976
|
-
projections: pattern.projections,
|
|
1977
|
-
presumptiveInventory: pattern.presumptive ?? true,
|
|
1978
|
-
path,
|
|
1979
|
-
line: index + 1,
|
|
1980
|
-
anchor: line,
|
|
1981
|
-
owner,
|
|
1982
|
-
identity: `${token}@${index + 1}:${match.index}`,
|
|
1983
|
-
tokens: [token],
|
|
1984
|
-
operationToken: token,
|
|
1985
|
-
});
|
|
1986
|
-
if (match[0].length === 0)
|
|
1987
|
-
pattern.regex.lastIndex += 1;
|
|
1988
|
-
}
|
|
1989
|
-
}
|
|
1990
|
-
const envMatches = [
|
|
1991
|
-
...line.matchAll(/os\.getenv\(\s*['"]([^'"]+)['"]/g),
|
|
1992
|
-
...line.matchAll(/os\.environ\.(get|pop|setdefault)\(\s*['"]([^'"]+)['"]/g),
|
|
1993
|
-
...line.matchAll(/os\.environ\[\s*['"]([^'"]+)['"]\s*\]/g),
|
|
1994
|
-
...line.matchAll(/process\.env\.([A-Za-z_][A-Za-z0-9_]*)/g),
|
|
1995
|
-
...line.matchAll(/process\.env\[\s*['"]([^'"]+)['"]\s*\]/g),
|
|
1996
|
-
].sort((left, right) => (left.index ?? 0) - (right.index ?? 0));
|
|
1997
|
-
for (const envMatch of envMatches) {
|
|
1998
|
-
const method = envMatch[1] && ['get', 'pop', 'setdefault'].includes(envMatch[1]) ? envMatch[1] : null;
|
|
1999
|
-
const key = method ? envMatch[2] : envMatch[1];
|
|
2000
|
-
const suffix = line.slice((envMatch.index ?? 0) + envMatch[0].length);
|
|
2001
|
-
const written = method === 'pop' || method === 'setdefault' || /^\s*=/.test(suffix);
|
|
2002
|
-
addRecord(records, {
|
|
2003
|
-
kind: written ? 'environment-write-operation' : 'environment-read-operation',
|
|
2004
|
-
projections: written ? ['stateful_resources', 'external_dependencies'] : ['external_dependencies'],
|
|
2005
|
-
presumptiveInventory: true,
|
|
2006
|
-
path, line: index + 1, anchor: line, owner,
|
|
2007
|
-
identity: `environment:${written ? 'write' : 'read'}:${key}@${index + 1}`,
|
|
2008
|
-
tokens: [key], operationToken: written ? 'environment-write' : 'environment-read', addressToken: key,
|
|
2009
|
-
});
|
|
2010
|
-
}
|
|
2011
|
-
const dynamicEnvMatches = [
|
|
2012
|
-
...line.matchAll(/os\.getenv\(\s*([^,)]+)(?:,|\))/g),
|
|
2013
|
-
...line.matchAll(/os\.environ\.(get|pop|setdefault)\(\s*([^,)]+)(?:,|\))/g),
|
|
2014
|
-
...line.matchAll(/os\.environ\[\s*([^\]]+)\s*\]/g),
|
|
2015
|
-
].sort((left, right) => (left.index ?? 0) - (right.index ?? 0));
|
|
2016
|
-
for (const envMatch of dynamicEnvMatches) {
|
|
2017
|
-
const method = envMatch[1] && ['get', 'pop', 'setdefault'].includes(envMatch[1])
|
|
2018
|
-
? envMatch[1]
|
|
2019
|
-
: null;
|
|
2020
|
-
const expression = (method ? envMatch[2] : envMatch[1])?.trim();
|
|
2021
|
-
if (!expression || /^['"][^'"]+['"]$/.test(expression))
|
|
2022
|
-
continue;
|
|
2023
|
-
const suffix = line.slice((envMatch.index ?? 0) + envMatch[0].length);
|
|
2024
|
-
const written = method === 'pop' || method === 'setdefault' || /^\s*=/.test(suffix);
|
|
2025
|
-
addRecord(records, {
|
|
2026
|
-
kind: written ? 'environment-write-operation' : 'environment-read-operation',
|
|
2027
|
-
projections: written ? ['stateful_resources', 'external_dependencies'] : ['external_dependencies'],
|
|
2028
|
-
presumptiveInventory: true,
|
|
2029
|
-
path, line: index + 1, anchor: line, owner,
|
|
2030
|
-
identity: `environment:${written ? 'write' : 'read'}:${expression}@${index + 1}`,
|
|
2031
|
-
tokens: [expression], operationToken: written ? 'environment-write' : 'environment-read',
|
|
2032
|
-
addressToken: expression,
|
|
2033
|
-
});
|
|
2034
|
-
}
|
|
2035
|
-
const synchronizedTrait = line.match(/^\s*([A-Za-z_][A-Za-z0-9_]*)\s*(?::[^=]+)?=.+\.tag\([^)]*\bsync\s*=\s*(?:True|true)\b/);
|
|
2036
|
-
if (synchronizedTrait)
|
|
2037
|
-
addRecord(records, {
|
|
2038
|
-
kind: 'synchronized-trait-state',
|
|
2039
|
-
projections: ['stateful_resources'],
|
|
2040
|
-
presumptiveInventory: true,
|
|
2041
|
-
path,
|
|
2042
|
-
line: index + 1,
|
|
2043
|
-
anchor: line,
|
|
2044
|
-
owner,
|
|
2045
|
-
identity: `synchronized-trait:${synchronizedTrait[1]}@${index + 1}`,
|
|
2046
|
-
tokens: [synchronizedTrait[1], 'sync'],
|
|
2047
|
-
addressToken: synchronizedTrait[1],
|
|
2048
|
-
operationToken: 'sync-trait',
|
|
2049
|
-
});
|
|
2050
|
-
for (const resourceAssignment of line.matchAll(/\b([A-Za-z_][A-Za-z0-9_.]*)\.(src|href)\s*=\s*([^;]+)/g))
|
|
2051
|
-
addRecord(records, {
|
|
2052
|
-
kind: 'browser-resource-load-operation',
|
|
2053
|
-
projections: ['external_dependencies'],
|
|
2054
|
-
presumptiveInventory: true,
|
|
2055
|
-
path,
|
|
2056
|
-
line: index + 1,
|
|
2057
|
-
anchor: line,
|
|
2058
|
-
owner,
|
|
2059
|
-
identity: `browser-resource:${resourceAssignment[1]}.${resourceAssignment[2]}:${resourceAssignment[3].trim()}@${index + 1}`,
|
|
2060
|
-
tokens: [resourceAssignment[1], resourceAssignment[2], resourceAssignment[3].trim()],
|
|
2061
|
-
operationToken: `${resourceAssignment[1]}.${resourceAssignment[2]}=`,
|
|
2062
|
-
addressToken: resourceAssignment[3].trim(),
|
|
2063
|
-
});
|
|
2064
|
-
const openWrite = line.match(/\bopen\s*\([^,\n]+,\s*(?:mode\s*=\s*)?['"]([wax+][^'"]*)['"]/) ??
|
|
2065
|
-
line.match(/\b[A-Za-z_][A-Za-z0-9_.]*\.open\s*\(\s*['"]([wax+][^'"]*)['"]/);
|
|
2066
|
-
if (openWrite)
|
|
2067
|
-
addRecord(records, {
|
|
2068
|
-
kind: 'filesystem-write-operation', projections: ['stateful_resources', 'external_dependencies'], presumptiveInventory: true,
|
|
2069
|
-
path, line: index + 1, anchor: line, owner,
|
|
2070
|
-
identity: `open-write:${openWrite[1]}@${index + 1}`, tokens: [openWrite[1]], operationToken: 'open-write',
|
|
2071
|
-
});
|
|
2072
|
-
for (const mutation of line.matchAll(/\b((?:self\.)?[A-Za-z_][A-Za-z0-9_]*(?:\.[A-Za-z_][A-Za-z0-9_]*)*)\.(append|extend|add|set|update|pop|remove|discard|clear|put|put_nowait|setdefault)\s*\(/g)) {
|
|
2073
|
-
const target = mutation[1];
|
|
2074
|
-
if (!target.startsWith('self.') &&
|
|
2075
|
-
!/^[A-Z][A-Z0-9_]*$/.test(target) &&
|
|
2076
|
-
!/(?:cache|config|credential|hook|open_span|queue|registry|session|state|store|token|variable)/i.test(target))
|
|
2077
|
-
continue;
|
|
2078
|
-
addRecord(records, {
|
|
2079
|
-
kind: 'in-memory-state-mutation', projections: ['stateful_resources'], presumptiveInventory: true,
|
|
2080
|
-
path, line: index + 1, anchor: line, owner,
|
|
2081
|
-
identity: `memory-mutation:${target}.${mutation[2]}@${index + 1}`,
|
|
2082
|
-
tokens: [target, mutation[2]], operationToken: mutation[2], addressToken: target,
|
|
2083
|
-
});
|
|
2084
|
-
}
|
|
2085
|
-
const indexedMutation = line.match(/^\s*((?:self\.)?[A-Za-z_][A-Za-z0-9_]*(?:\.[A-Za-z_][A-Za-z0-9_]*)*)\s*\[[^\]]+\]\s*=/);
|
|
2086
|
-
if (indexedMutation &&
|
|
2087
|
-
(indexedMutation[1].startsWith('self.') ||
|
|
2088
|
-
/^[A-Z][A-Z0-9_]*$/.test(indexedMutation[1]) ||
|
|
2089
|
-
/(?:cache|config|credential|queue|registry|session|state|store|token|variable)/i.test(indexedMutation[1])))
|
|
2090
|
-
addRecord(records, {
|
|
2091
|
-
kind: 'in-memory-state-mutation', projections: ['stateful_resources'], presumptiveInventory: true,
|
|
2092
|
-
path, line: index + 1, anchor: line, owner,
|
|
2093
|
-
identity: `memory-index-write:${indexedMutation[1]}@${index + 1}`,
|
|
2094
|
-
tokens: [indexedMutation[1]], operationToken: 'indexed-write', addressToken: indexedMutation[1],
|
|
2095
|
-
});
|
|
2096
|
-
for (const remoteMutation of line.matchAll(/\b((?:[A-Za-z_][A-Za-z0-9_]*\.)+[A-Za-z_][A-Za-z0-9_]*)\.(post|put|patch|delete|create|update|remove|publish)\s*\(/gi)) {
|
|
2097
|
-
addRecord(records, {
|
|
2098
|
-
kind: 'remote-state-mutation-operation', projections: ['stateful_resources', 'external_dependencies'], presumptiveInventory: true,
|
|
2099
|
-
path, line: index + 1, anchor: line, owner,
|
|
2100
|
-
identity: `remote-mutation:${remoteMutation[1]}.${remoteMutation[2]}@${index + 1}`,
|
|
2101
|
-
tokens: [remoteMutation[1], remoteMutation[2]], operationToken: remoteMutation[2].toLowerCase(),
|
|
2102
|
-
});
|
|
2103
|
-
}
|
|
2104
|
-
const stateAddress = line.match(/^\s*((?:self\.)?[A-Za-z_][A-Za-z0-9_]*(?:PATH|FILE|DIR|HOME|CACHE|CONFIG|STORE|TOKEN|CREDENTIAL|DATABASE)[A-Za-z0-9_]*)\s*(?::[^=]+)?=\s*(.+)$/i);
|
|
2105
|
-
if (stateAddress && /(?:\bPath\s*\(|\/\s*['"]|['"][^'"]*(?:\/|\.[A-Za-z0-9]{1,8})['"])/.test(stateAddress[2])) {
|
|
2106
|
-
addRecord(records, {
|
|
2107
|
-
kind: 'state-address-declaration', projections: ['stateful_resources'], presumptiveInventory: true,
|
|
2108
|
-
path, line: index + 1, anchor: line, owner,
|
|
2109
|
-
identity: `state-address:${stateAddress[1]}@${index + 1}`,
|
|
2110
|
-
tokens: [stateAddress[1], stateAddress[2].trim()], addressToken: stateAddress[1],
|
|
2111
|
-
});
|
|
2112
|
-
}
|
|
2113
|
-
const dataclassState = line.match(/^\s*([A-Za-z_][A-Za-z0-9_]*)\s*:\s*[^=]+?=\s*field\(\s*default_factory\s*=\s*([A-Za-z_][A-Za-z0-9_.]*)/);
|
|
2114
|
-
if (dataclassState)
|
|
2115
|
-
addRecord(records, {
|
|
2116
|
-
kind: 'in-memory-state', projections: ['stateful_resources'], presumptiveInventory: true,
|
|
2117
|
-
path, line: index + 1, anchor: line, owner,
|
|
2118
|
-
identity: `dataclass-state:${dataclassState[1]}:${dataclassState[2]}@${index + 1}`,
|
|
2119
|
-
tokens: [dataclassState[1], dataclassState[2]], addressToken: dataclassState[1],
|
|
2120
|
-
});
|
|
2121
|
-
const mutableAssignment = line.match(/^\s*(?:self\.)?([A-Za-z_][A-Za-z0-9_]*)\s*(?::[^=]+)?=\s*(\{\}|\[\]|set\(\)|dict\(\)|list\(\))/);
|
|
2122
|
-
if (mutableAssignment)
|
|
2123
|
-
addRecord(records, {
|
|
2124
|
-
kind: 'in-memory-state', projections: ['stateful_resources'], presumptiveInventory: false,
|
|
2125
|
-
path, line: index + 1, anchor: line, owner,
|
|
2126
|
-
identity: `mutable:${mutableAssignment[1]}@${index + 1}`, tokens: [mutableAssignment[1], mutableAssignment[2]],
|
|
2127
|
-
addressToken: mutableAssignment[1],
|
|
2128
|
-
});
|
|
2129
|
-
const wrapperAssignment = line.match(/^\s*([A-Za-z_][A-Za-z0-9_.]*)\s*=\s*([A-Za-z_][A-Za-z0-9_.]*)\s*(?:#.*)?$/);
|
|
2130
|
-
const wrapperRoot = wrapperAssignment?.[1].split('.')[0] ?? '';
|
|
2131
|
-
const wrapperTarget = wrapperAssignment?.[2] ?? '';
|
|
2132
|
-
const looksLikeInstalledWrapper = Boolean(wrapperAssignment
|
|
2133
|
-
&& wrapperAssignment[1].includes('.')
|
|
2134
|
-
&& (/^[A-Z]/.test(wrapperRoot)
|
|
2135
|
-
|| /^(?:builtins|os|sys|asyncio|concurrent|threading|socket|signal)\./.test(wrapperAssignment[1])
|
|
2136
|
-
|| /(?:wrap|patch|handler|hook|instrument|send|receive|respond)/i.test(wrapperTarget)));
|
|
2137
|
-
if (wrapperAssignment && looksLikeInstalledWrapper)
|
|
2138
|
-
addRecord(records, {
|
|
2139
|
-
kind: 'installed-wrapper-state', projections: ['stateful_resources', 'external_dependencies'], presumptiveInventory: true,
|
|
2140
|
-
path, line: index + 1, anchor: line, owner,
|
|
2141
|
-
identity: `patch:${wrapperAssignment[1]}:${wrapperAssignment[2]}@${index + 1}`,
|
|
2142
|
-
tokens: [wrapperAssignment[1], wrapperAssignment[2]], operationToken: wrapperAssignment[1], addressToken: wrapperAssignment[1],
|
|
2143
|
-
});
|
|
2144
|
-
const routeMethods = [...line.matchAll(/['"](GET|POST|PUT|PATCH|DELETE|OPTIONS|HEAD)['"]/g)].map((match) => match[1]);
|
|
2145
|
-
if (/\b(?:Route|router\.|app\.|add_url_rule)/.test(line) && routeMethods.length > 0) {
|
|
2146
|
-
const route = line.match(/(?:f|r|fr|rf)?['"](\/[^'"]*)['"]/)?.[1];
|
|
2147
|
-
routeMethods.forEach((method) => addRecord(records, {
|
|
2148
|
-
kind: 'required-inbound-operation', projections: ['external_dependencies'], presumptiveInventory: true,
|
|
2149
|
-
path, line: index + 1, anchor: line, owner,
|
|
2150
|
-
identity: `inbound:${method}:${route ?? 'route'}@${index + 1}`,
|
|
2151
|
-
tokens: [method, route ?? 'route'], operationToken: method, addressToken: route,
|
|
2152
|
-
}));
|
|
2153
|
-
}
|
|
2154
|
-
});
|
|
2155
|
-
}
|
|
2156
|
-
function parenthesisDelta(value) {
|
|
2157
|
-
let delta = 0;
|
|
2158
|
-
let quote = null;
|
|
2159
|
-
let escaped = false;
|
|
2160
|
-
for (const char of value) {
|
|
2161
|
-
if (escaped) {
|
|
2162
|
-
escaped = false;
|
|
2163
|
-
continue;
|
|
2164
|
-
}
|
|
2165
|
-
if (char === '\\') {
|
|
2166
|
-
escaped = true;
|
|
2167
|
-
continue;
|
|
2168
|
-
}
|
|
2169
|
-
if (quote) {
|
|
2170
|
-
if (char === quote)
|
|
2171
|
-
quote = null;
|
|
2172
|
-
continue;
|
|
2173
|
-
}
|
|
2174
|
-
if (char === "'" || char === '"') {
|
|
2175
|
-
quote = char;
|
|
2176
|
-
continue;
|
|
2177
|
-
}
|
|
2178
|
-
if (char === '(')
|
|
2179
|
-
delta += 1;
|
|
2180
|
-
if (char === ')')
|
|
2181
|
-
delta -= 1;
|
|
2182
|
-
}
|
|
2183
|
-
return delta;
|
|
2184
|
-
}
|
|
2185
|
-
function splitTopLevelJavaScript(value) {
|
|
2186
|
-
const segments = [];
|
|
2187
|
-
let start = 0;
|
|
2188
|
-
let parentheses = 0;
|
|
2189
|
-
let brackets = 0;
|
|
2190
|
-
let braces = 0;
|
|
2191
|
-
let quote = null;
|
|
2192
|
-
let escaped = false;
|
|
2193
|
-
let lineComment = false;
|
|
2194
|
-
let blockComment = false;
|
|
2195
|
-
for (let index = 0; index < value.length; index += 1) {
|
|
2196
|
-
const char = value[index];
|
|
2197
|
-
const next = value[index + 1];
|
|
2198
|
-
if (lineComment) {
|
|
2199
|
-
if (char === '\n')
|
|
2200
|
-
lineComment = false;
|
|
2201
|
-
continue;
|
|
2202
|
-
}
|
|
2203
|
-
if (blockComment) {
|
|
2204
|
-
if (char === '*' && next === '/') {
|
|
2205
|
-
blockComment = false;
|
|
2206
|
-
index += 1;
|
|
2207
|
-
}
|
|
2208
|
-
continue;
|
|
2209
|
-
}
|
|
2210
|
-
if (escaped) {
|
|
2211
|
-
escaped = false;
|
|
2212
|
-
continue;
|
|
2213
|
-
}
|
|
2214
|
-
if (quote) {
|
|
2215
|
-
if (char === '\\')
|
|
2216
|
-
escaped = true;
|
|
2217
|
-
else if (char === quote)
|
|
2218
|
-
quote = null;
|
|
2219
|
-
continue;
|
|
2220
|
-
}
|
|
2221
|
-
if (char === '/' && next === '/') {
|
|
2222
|
-
lineComment = true;
|
|
2223
|
-
index += 1;
|
|
2224
|
-
continue;
|
|
2225
|
-
}
|
|
2226
|
-
if (char === '/' && next === '*') {
|
|
2227
|
-
blockComment = true;
|
|
2228
|
-
index += 1;
|
|
2229
|
-
continue;
|
|
2230
|
-
}
|
|
2231
|
-
if (char === "'" || char === '"' || char === '`') {
|
|
2232
|
-
quote = char;
|
|
2233
|
-
continue;
|
|
2234
|
-
}
|
|
2235
|
-
if (char === '(')
|
|
2236
|
-
parentheses += 1;
|
|
2237
|
-
else if (char === ')')
|
|
2238
|
-
parentheses -= 1;
|
|
2239
|
-
else if (char === '[')
|
|
2240
|
-
brackets += 1;
|
|
2241
|
-
else if (char === ']')
|
|
2242
|
-
brackets -= 1;
|
|
2243
|
-
else if (char === '{')
|
|
2244
|
-
braces += 1;
|
|
2245
|
-
else if (char === '}')
|
|
2246
|
-
braces -= 1;
|
|
2247
|
-
else if (char === ',' && parentheses === 0 && brackets === 0 && braces === 0) {
|
|
2248
|
-
segments.push(value.slice(start, index));
|
|
2249
|
-
start = index + 1;
|
|
2250
|
-
}
|
|
2251
|
-
}
|
|
2252
|
-
segments.push(value.slice(start));
|
|
2253
|
-
return segments;
|
|
2254
|
-
}
|
|
2255
|
-
function javascriptCallArguments(call) {
|
|
2256
|
-
const open = call.indexOf('(');
|
|
2257
|
-
if (open < 0)
|
|
2258
|
-
return [];
|
|
2259
|
-
let depth = 0;
|
|
2260
|
-
let quote = null;
|
|
2261
|
-
let escaped = false;
|
|
2262
|
-
let lineComment = false;
|
|
2263
|
-
let blockComment = false;
|
|
2264
|
-
for (let index = open + 1; index < call.length; index += 1) {
|
|
2265
|
-
const char = call[index];
|
|
2266
|
-
const next = call[index + 1];
|
|
2267
|
-
if (lineComment) {
|
|
2268
|
-
if (char === '\n')
|
|
2269
|
-
lineComment = false;
|
|
2270
|
-
continue;
|
|
2271
|
-
}
|
|
2272
|
-
if (blockComment) {
|
|
2273
|
-
if (char === '*' && next === '/') {
|
|
2274
|
-
blockComment = false;
|
|
2275
|
-
index += 1;
|
|
2276
|
-
}
|
|
2277
|
-
continue;
|
|
2278
|
-
}
|
|
2279
|
-
if (escaped) {
|
|
2280
|
-
escaped = false;
|
|
2281
|
-
continue;
|
|
2282
|
-
}
|
|
2283
|
-
if (quote) {
|
|
2284
|
-
if (char === '\\')
|
|
2285
|
-
escaped = true;
|
|
2286
|
-
else if (char === quote)
|
|
2287
|
-
quote = null;
|
|
2288
|
-
continue;
|
|
2289
|
-
}
|
|
2290
|
-
if (char === '/' && next === '/') {
|
|
2291
|
-
lineComment = true;
|
|
2292
|
-
index += 1;
|
|
2293
|
-
continue;
|
|
2294
|
-
}
|
|
2295
|
-
if (char === '/' && next === '*') {
|
|
2296
|
-
blockComment = true;
|
|
2297
|
-
index += 1;
|
|
2298
|
-
continue;
|
|
2299
|
-
}
|
|
2300
|
-
if (char === "'" || char === '"' || char === '`') {
|
|
2301
|
-
quote = char;
|
|
2302
|
-
continue;
|
|
2303
|
-
}
|
|
2304
|
-
if (char === '(')
|
|
2305
|
-
depth += 1;
|
|
2306
|
-
else if (char === ')' && depth > 0)
|
|
2307
|
-
depth -= 1;
|
|
2308
|
-
else if (char === ')')
|
|
2309
|
-
return splitTopLevelJavaScript(call.slice(open + 1, index));
|
|
2310
|
-
}
|
|
2311
|
-
return [];
|
|
2312
|
-
}
|
|
2313
|
-
function directFetchMethod(call) {
|
|
2314
|
-
const init = javascriptCallArguments(call)[1]?.trim();
|
|
2315
|
-
if (!init?.startsWith('{'))
|
|
2316
|
-
return undefined;
|
|
2317
|
-
let close = -1;
|
|
2318
|
-
let depth = 0;
|
|
2319
|
-
let quote = null;
|
|
2320
|
-
let escaped = false;
|
|
2321
|
-
for (let index = 0; index < init.length; index += 1) {
|
|
2322
|
-
const char = init[index];
|
|
2323
|
-
if (escaped) {
|
|
2324
|
-
escaped = false;
|
|
2325
|
-
continue;
|
|
2326
|
-
}
|
|
2327
|
-
if (quote) {
|
|
2328
|
-
if (char === '\\')
|
|
2329
|
-
escaped = true;
|
|
2330
|
-
else if (char === quote)
|
|
2331
|
-
quote = null;
|
|
2332
|
-
continue;
|
|
2333
|
-
}
|
|
2334
|
-
if (char === "'" || char === '"' || char === '`') {
|
|
2335
|
-
quote = char;
|
|
2336
|
-
continue;
|
|
2337
|
-
}
|
|
2338
|
-
if (char === '{')
|
|
2339
|
-
depth += 1;
|
|
2340
|
-
else if (char === '}') {
|
|
2341
|
-
depth -= 1;
|
|
2342
|
-
if (depth === 0) {
|
|
2343
|
-
close = index;
|
|
2344
|
-
break;
|
|
2345
|
-
}
|
|
2346
|
-
}
|
|
2347
|
-
}
|
|
2348
|
-
if (close < 0)
|
|
2349
|
-
return undefined;
|
|
2350
|
-
let method;
|
|
2351
|
-
for (const property of splitTopLevelJavaScript(init.slice(1, close))) {
|
|
2352
|
-
const direct = property.match(/^\s*(?:method|['"]method['"])\s*:\s*([\s\S]*)$/i);
|
|
2353
|
-
if (!direct)
|
|
2354
|
-
continue;
|
|
2355
|
-
method = direct[1].match(/^\s*(['"])(GET|POST|PUT|PATCH|DELETE|OPTIONS|HEAD)\1(?:\s|$)/i)?.[2];
|
|
2356
|
-
}
|
|
2357
|
-
return method;
|
|
2358
|
-
}
|
|
2359
|
-
function scanExternalCallBlocks(path, lines, owners, executable, records) {
|
|
2360
|
-
for (let index = 0; index < lines.length; index += 1) {
|
|
2361
|
-
if (!executable[index])
|
|
2362
|
-
continue;
|
|
2363
|
-
const start = lines[index];
|
|
2364
|
-
const call = start.match(/\b([A-Za-z_][A-Za-z0-9_.]*)\s*\(/);
|
|
2365
|
-
if (!call)
|
|
2366
|
-
continue;
|
|
2367
|
-
let depth = parenthesisDelta(start.slice(call.index ?? 0));
|
|
2368
|
-
const block = [start];
|
|
2369
|
-
let end = index;
|
|
2370
|
-
while (depth > 0 && end + 1 < lines.length && end - index < 40) {
|
|
2371
|
-
end += 1;
|
|
2372
|
-
block.push(lines[end]);
|
|
2373
|
-
depth += parenthesisDelta(lines[end]);
|
|
2374
|
-
}
|
|
2375
|
-
const blockText = block.join('\n');
|
|
2376
|
-
const callable = call[1];
|
|
2377
|
-
const shortCallable = callable.split('.').at(-1) ?? callable;
|
|
2378
|
-
const owner = owners[index] ?? basename(path);
|
|
2379
|
-
const callLiterals = literalTokens(blockText);
|
|
2380
|
-
if (shortCallable === 'register' && callLiterals.length > 0) {
|
|
2381
|
-
const selector = callLiterals[0];
|
|
2382
|
-
const argumentText = blockText.slice(blockText.indexOf('(') + 1, blockText.lastIndexOf(')'));
|
|
2383
|
-
const implementation = argumentText.split(',').slice(1).join(',').trim() || 'registered-callback';
|
|
2384
|
-
addRecord(records, {
|
|
2385
|
-
kind: 'installed-registration-state',
|
|
2386
|
-
projections: ['stateful_resources'],
|
|
2387
|
-
presumptiveInventory: true,
|
|
2388
|
-
path,
|
|
2389
|
-
line: index + 1,
|
|
2390
|
-
anchor: start,
|
|
2391
|
-
owner,
|
|
2392
|
-
identity: `installed-registration:${callable}:${selector}:${implementation}@${index + 1}`,
|
|
2393
|
-
tokens: [callable, selector, implementation],
|
|
2394
|
-
selectorKey: selector,
|
|
2395
|
-
implementationKey: implementation,
|
|
2396
|
-
operationToken: callable,
|
|
2397
|
-
addressToken: selector,
|
|
2398
|
-
});
|
|
2399
|
-
}
|
|
2400
|
-
if (/^(?:addEventListener|addListener|observe|on|register_magic_function|subscribe|watch)$/.test(shortCallable)) {
|
|
2401
|
-
const selector = callLiterals.join('|') || 'all-events';
|
|
2402
|
-
addRecord(records, {
|
|
2403
|
-
kind: 'callback-registration-operation',
|
|
2404
|
-
projections: path.endsWith('.py')
|
|
2405
|
-
? ['stateful_resources', 'external_dependencies']
|
|
2406
|
-
: ['external_dependencies'],
|
|
2407
|
-
presumptiveInventory: true,
|
|
2408
|
-
path,
|
|
2409
|
-
line: index + 1,
|
|
2410
|
-
anchor: start,
|
|
2411
|
-
owner,
|
|
2412
|
-
identity: `callback-registration:${callable}:${selector}@${index + 1}`,
|
|
2413
|
-
tokens: [callable, ...callLiterals],
|
|
2414
|
-
selectorKey: selector,
|
|
2415
|
-
implementationKey: `callback@${path}:${index + 1}`,
|
|
2416
|
-
operationToken: callable,
|
|
2417
|
-
addressToken: selector,
|
|
2418
|
-
});
|
|
2419
|
-
}
|
|
2420
|
-
const constructorBoundary = /(?:Api|Channel|Client|Connection|Consumer|Exporter|Producer|Provider|Publisher|Reader|Session|Stub|Transport|Writer)$/i.test(shortCallable) &&
|
|
2421
|
-
/(?:endpoint|base_url|url|host|address|dsn|channel|connection|session|transport|headers|credentials)\s*=/i.test(blockText);
|
|
2422
|
-
if (constructorBoundary) {
|
|
2423
|
-
addRecord(records, {
|
|
2424
|
-
kind: 'external-constructor-operation', projections: ['external_dependencies'], presumptiveInventory: true,
|
|
2425
|
-
path, line: index + 1, anchor: start, owner,
|
|
2426
|
-
identity: `external-constructor:${callable}@${index + 1}`,
|
|
2427
|
-
tokens: [callable, ...literalTokens(blockText).slice(0, 8)],
|
|
2428
|
-
operationToken: callable,
|
|
2429
|
-
});
|
|
2430
|
-
}
|
|
2431
|
-
if (/(?:^|\.)(?:run|Popen|call|check_call|check_output|spawn|exec|execFile)$/.test(callable) &&
|
|
2432
|
-
/['"](?:add|configure|create|delete|disable|enable|install|patch|register|remove|set|update)['"]/.test(blockText)) {
|
|
2433
|
-
addRecord(records, {
|
|
2434
|
-
kind: 'installed-command-state', projections: ['stateful_resources', 'external_dependencies'], presumptiveInventory: true,
|
|
2435
|
-
path, line: index + 1, anchor: start, owner,
|
|
2436
|
-
identity: `installed-command:${callable}:${literalTokens(blockText).join(':')}@${index + 1}`,
|
|
2437
|
-
tokens: [callable, ...literalTokens(blockText).slice(0, 10)], operationToken: callable,
|
|
2438
|
-
});
|
|
2439
|
-
}
|
|
2440
|
-
if (/^(?:self\.)?_(?:get|post|put|patch|delete|request)(?:_raw)?$/i.test(callable)) {
|
|
2441
|
-
const address = blockText.match(/(?:f|r|fr|rf)?['"]([^'"]*(?:\/|https?:)[^'"]*)['"]/)?.[1];
|
|
2442
|
-
if (address)
|
|
2443
|
-
addRecord(records, {
|
|
2444
|
-
kind: 'http-wrapper-operation',
|
|
2445
|
-
projections: /_(?:post|put|patch|delete)(?:_raw)?$/i.test(callable)
|
|
2446
|
-
? ['stateful_resources', 'external_dependencies']
|
|
2447
|
-
: ['external_dependencies'],
|
|
2448
|
-
presumptiveInventory: true,
|
|
2449
|
-
path, line: index + 1, anchor: start, owner,
|
|
2450
|
-
identity: `http-wrapper:${callable}:${address}@${index + 1}`,
|
|
2451
|
-
tokens: [callable, address], operationToken: callable, addressToken: address,
|
|
2452
|
-
});
|
|
2453
|
-
}
|
|
2454
|
-
if (/(?:^|\.)(?:Route|route|add_url_rule)$/.test(callable)) {
|
|
2455
|
-
const methods = [...blockText.matchAll(/['"](GET|POST|PUT|PATCH|DELETE|OPTIONS|HEAD)['"]/g)]
|
|
2456
|
-
.map((match) => match[1]);
|
|
2457
|
-
const route = blockText.match(/(?:f|r|fr|rf)?['"](\/[^'"]*)['"]/)?.[1];
|
|
2458
|
-
[...new Set(methods)].forEach((method) => addRecord(records, {
|
|
2459
|
-
kind: 'required-inbound-operation', projections: ['external_dependencies'], presumptiveInventory: true,
|
|
2460
|
-
path, line: index + 1, anchor: start, owner,
|
|
2461
|
-
identity: `inbound:${method}:${route ?? callable}@${index + 1}`,
|
|
2462
|
-
tokens: [method, route ?? callable], operationToken: method, addressToken: route,
|
|
2463
|
-
}));
|
|
2464
|
-
}
|
|
2465
|
-
}
|
|
2466
|
-
}
|
|
2467
|
-
function scanPythonProtocolCallbacks(path, lines, owners, executable, records) {
|
|
2468
|
-
lines.forEach((line, index) => {
|
|
2469
|
-
if (!executable[index])
|
|
2470
|
-
return;
|
|
2471
|
-
const method = line.match(/^\s*(?:async\s+def|def)\s+([A-Za-z_][A-Za-z0-9_]*)\s*\(/)?.[1];
|
|
2472
|
-
if (!method || /^__.*__$/.test(method))
|
|
2473
|
-
return;
|
|
2474
|
-
const owner = owners[index] ?? basename(path);
|
|
2475
|
-
const classOwner = owner.split('.').slice(0, -1).join('.');
|
|
2476
|
-
if (!/(?:Api|Callback|Client|Consumer|Executor|Exporter|Finder|Handler|Hook|Instrumentor|Listener|Loader|Plugin|Processor|Provider|Publisher|Receiver|Session|Transport|Wrapper)(?:\.|$)/i.test(classOwner))
|
|
2477
|
-
return;
|
|
2478
|
-
const projections = ['external_dependencies'];
|
|
2479
|
-
if (/(?:Callback|Finder|Handler|Hook|Instrumentor|Loader|Plugin|Wrapper)(?:\.|$)/i.test(classOwner)) {
|
|
2480
|
-
projections.push('stateful_resources');
|
|
2481
|
-
}
|
|
2482
|
-
addRecord(records, {
|
|
2483
|
-
kind: 'protocol-callback-operation', projections, presumptiveInventory: true,
|
|
2484
|
-
path, line: index + 1, anchor: line, owner: classOwner,
|
|
2485
|
-
identity: `protocol-callback:${classOwner}.${method}@${index + 1}`,
|
|
2486
|
-
tokens: [classOwner, method], operationToken: method,
|
|
2487
|
-
});
|
|
2488
|
-
});
|
|
2489
|
-
}
|
|
2490
|
-
function scanPythonOptionalImports(path, lines, owners, executable, records) {
|
|
2491
|
-
lines.forEach((line, index) => {
|
|
2492
|
-
if (!executable[index])
|
|
2493
|
-
return;
|
|
2494
|
-
const imported = line.match(/^\s*(?:from\s+([A-Za-z_][A-Za-z0-9_.]*)\s+import|import\s+([A-Za-z_][A-Za-z0-9_.]*))/);
|
|
2495
|
-
if (!imported)
|
|
2496
|
-
return;
|
|
2497
|
-
const owner = owners[index] ?? basename(path);
|
|
2498
|
-
const nearby = lines.slice(Math.max(0, index - 3), Math.min(lines.length, index + 5)).join('\n');
|
|
2499
|
-
if (!/except\s+(?:ImportError|ModuleNotFoundError)|\btry\s*:|optional\s+(?:dependency|package)|required\s+to\s+use/i.test(nearby))
|
|
2500
|
-
return;
|
|
2501
|
-
const name = normalizePackageName((imported[1] ?? imported[2]).split('.')[0]);
|
|
2502
|
-
addRecord(records, {
|
|
2503
|
-
kind: 'optional-import-package-state', projections: ['stateful_resources'], presumptiveInventory: false,
|
|
2504
|
-
path, line: index + 1, anchor: line, owner,
|
|
2505
|
-
identity: `optional-import:${name}@${index + 1}`,
|
|
2506
|
-
tokens: [name], addressToken: name, packageName: name,
|
|
2507
|
-
});
|
|
2508
|
-
addRecord(records, {
|
|
2509
|
-
kind: 'optional-import-operation', projections: ['external_dependencies'], presumptiveInventory: false,
|
|
2510
|
-
path, line: index + 1, anchor: line, owner,
|
|
2511
|
-
identity: `optional-import:${name}@${index + 1}`,
|
|
2512
|
-
tokens: [name], operationToken: 'import-optional-package', packageName: name,
|
|
2513
|
-
});
|
|
2514
|
-
});
|
|
2515
|
-
}
|
|
2516
|
-
const PYTHON_STANDARD_LIBRARY_ROOTS = new Set([
|
|
2517
|
-
'__future__', 'abc', 'argparse', 'array', 'ast', 'asyncio', 'atexit', 'base64',
|
|
2518
|
-
'bisect', 'builtins', 'bz2', 'calendar', 'cmath', 'cmd', 'code', 'codecs',
|
|
2519
|
-
'collections', 'concurrent', 'configparser', 'contextlib', 'contextvars', 'copy',
|
|
2520
|
-
'csv', 'ctypes', 'dataclasses', 'datetime', 'decimal', 'difflib', 'dis', 'email',
|
|
2521
|
-
'enum', 'errno', 'faulthandler', 'fcntl', 'filecmp', 'fileinput', 'fnmatch',
|
|
2522
|
-
'fractions', 'ftplib', 'functools', 'gc', 'getopt', 'getpass', 'gettext', 'glob',
|
|
2523
|
-
'graphlib', 'gzip', 'hashlib', 'heapq', 'hmac', 'html', 'http', 'imaplib',
|
|
2524
|
-
'importlib', 'inspect', 'io', 'ipaddress', 'itertools', 'json', 'keyword',
|
|
2525
|
-
'linecache', 'locale', 'logging', 'lzma', 'mailbox', 'math', 'mimetypes',
|
|
2526
|
-
'multiprocessing', 'netrc', 'numbers', 'operator', 'os', 'pathlib', 'pickle',
|
|
2527
|
-
'pkgutil', 'platform', 'plistlib', 'pprint', 'profile', 'pstats', 'pty', 'pwd',
|
|
2528
|
-
'queue', 'random', 're', 'readline', 'resource', 'runpy', 'sched', 'secrets',
|
|
2529
|
-
'select', 'selectors', 'shelve', 'shlex', 'shutil', 'signal', 'site', 'smtplib',
|
|
2530
|
-
'socket', 'sqlite3', 'ssl', 'stat', 'statistics', 'string', 'stringprep',
|
|
2531
|
-
'struct', 'subprocess', 'sys', 'sysconfig', 'tarfile', 'tempfile', 'textwrap',
|
|
2532
|
-
'threading', 'time', 'timeit', 'tkinter', 'token', 'tokenize', 'tomllib',
|
|
2533
|
-
'trace', 'traceback', 'tracemalloc', 'types', 'typing', 'unittest', 'urllib',
|
|
2534
|
-
'uuid', 'venv', 'warnings', 'wave', 'weakref', 'webbrowser', 'xml', 'xmlrpc',
|
|
2535
|
-
'zipfile', 'zipimport', 'zlib', 'zoneinfo',
|
|
2536
|
-
]);
|
|
2537
|
-
function scanPythonImportedSdkOperations(path, lines, owners, executable, localModuleRoots, records) {
|
|
2538
|
-
const bindings = new Map();
|
|
2539
|
-
for (const line of lines) {
|
|
2540
|
-
const importedModule = line.match(/^\s*import\s+([A-Za-z_][A-Za-z0-9_.]*)(?:\s+as\s+([A-Za-z_][A-Za-z0-9_]*))?/);
|
|
2541
|
-
if (importedModule) {
|
|
2542
|
-
const root = importedModule[1].split('.')[0];
|
|
2543
|
-
if (!PYTHON_STANDARD_LIBRARY_ROOTS.has(root) && !localModuleRoots.has(root)) {
|
|
2544
|
-
bindings.set(importedModule[2] ?? root, importedModule[1]);
|
|
2545
|
-
}
|
|
2546
|
-
}
|
|
2547
|
-
const importedSymbols = line.match(/^\s*from\s+([A-Za-z_][A-Za-z0-9_.]*)\s+import\s+(.+)$/);
|
|
2548
|
-
if (!importedSymbols)
|
|
2549
|
-
continue;
|
|
2550
|
-
const root = importedSymbols[1].split('.')[0];
|
|
2551
|
-
if (PYTHON_STANDARD_LIBRARY_ROOTS.has(root) || localModuleRoots.has(root))
|
|
2552
|
-
continue;
|
|
2553
|
-
for (const rawSymbol of importedSymbols[2].replace(/[()]/g, '').split(',')) {
|
|
2554
|
-
const symbol = rawSymbol.trim().match(/^([A-Za-z_][A-Za-z0-9_]*)(?:\s+as\s+([A-Za-z_][A-Za-z0-9_]*))?/);
|
|
2555
|
-
if (symbol)
|
|
2556
|
-
bindings.set(symbol[2] ?? symbol[1], `${importedSymbols[1]}.${symbol[1]}`);
|
|
2557
|
-
}
|
|
2558
|
-
}
|
|
2559
|
-
if (bindings.size === 0)
|
|
2560
|
-
return;
|
|
2561
|
-
const instanceBindings = new Map();
|
|
2562
|
-
for (const line of lines) {
|
|
2563
|
-
const assignment = line.match(/^\s*([A-Za-z_][A-Za-z0-9_]*)\s*=\s*([A-Za-z_][A-Za-z0-9_.]*)\s*\(/);
|
|
2564
|
-
if (!assignment)
|
|
2565
|
-
continue;
|
|
2566
|
-
const root = assignment[2].split('.')[0];
|
|
2567
|
-
const importedIdentity = bindings.get(root);
|
|
2568
|
-
const constructor = assignment[2].split('.').at(-1) ?? '';
|
|
2569
|
-
if (importedIdentity &&
|
|
2570
|
-
/(?:Api|Browser|Channel|Client|Connection|Consumer|Exporter|Producer|Provider|Publisher|Reader|Resolver|Session|Stub|Transport|Writer)$/i.test(constructor)) {
|
|
2571
|
-
instanceBindings.set(assignment[1], `${importedIdentity}${assignment[2].slice(root.length)}`);
|
|
2572
|
-
}
|
|
2573
|
-
}
|
|
2574
|
-
lines.forEach((line, index) => {
|
|
2575
|
-
if (!executable[index] || /^\s*(?:from|import)\s/.test(line))
|
|
2576
|
-
return;
|
|
2577
|
-
const owner = owners[index] ?? basename(path);
|
|
2578
|
-
for (const [binding, importedIdentity] of bindings) {
|
|
2579
|
-
const escaped = binding.replace(/[.*+?^${}()|[\]\\]/g, '\\$&');
|
|
2580
|
-
const callPattern = new RegExp(`\\b(${escaped}(?:\\.[A-Za-z_][A-Za-z0-9_]*)*)\\s*\\(`, 'g');
|
|
2581
|
-
for (const match of line.matchAll(callPattern)) {
|
|
2582
|
-
const callable = match[1];
|
|
2583
|
-
const boundaryLike = /(?:api|browser|client|convert|display|entry.?point|export|fusion|host|http|loader|model|network|plugin|resource|schema|sdk|session|socket|trait|transport|widget)/i
|
|
2584
|
-
.test(`${importedIdentity} ${callable}`);
|
|
2585
|
-
addRecord(records, {
|
|
2586
|
-
kind: 'imported-sdk-operation',
|
|
2587
|
-
projections: ['external_dependencies'],
|
|
2588
|
-
presumptiveInventory: boundaryLike,
|
|
2589
|
-
path,
|
|
2590
|
-
line: index + 1,
|
|
2591
|
-
anchor: line,
|
|
2592
|
-
owner,
|
|
2593
|
-
identity: `imported-sdk:${importedIdentity}:${callable}@${index + 1}:${match.index ?? 0}`,
|
|
2594
|
-
tokens: [importedIdentity, callable],
|
|
2595
|
-
operationToken: callable,
|
|
2596
|
-
addressToken: importedIdentity,
|
|
2597
|
-
});
|
|
2598
|
-
}
|
|
2599
|
-
}
|
|
2600
|
-
for (const [binding, constructorIdentity] of instanceBindings) {
|
|
2601
|
-
const escaped = binding.replace(/[.*+?^${}()|[\]\\]/g, '\\$&');
|
|
2602
|
-
const callPattern = new RegExp(`\\b(${escaped}\\.([A-Za-z_][A-Za-z0-9_]*))\\s*\\(`, 'g');
|
|
2603
|
-
for (const match of line.matchAll(callPattern)) {
|
|
2604
|
-
const callable = match[1];
|
|
2605
|
-
const method = match[2];
|
|
2606
|
-
const boundaryLike = /^(?:ack|call|connect|consume|create|delete|execute|export|get|invoke|list|load|post|publish|put|query|read|receive|remove|request|respond|send|stream|update|write)$/i
|
|
2607
|
-
.test(method);
|
|
2608
|
-
addRecord(records, {
|
|
2609
|
-
kind: 'imported-sdk-instance-operation',
|
|
2610
|
-
projections: ['external_dependencies'],
|
|
2611
|
-
presumptiveInventory: boundaryLike,
|
|
2612
|
-
path,
|
|
2613
|
-
line: index + 1,
|
|
2614
|
-
anchor: line,
|
|
2615
|
-
owner,
|
|
2616
|
-
identity: `imported-sdk-instance:${constructorIdentity}:${callable}@${index + 1}:${match.index ?? 0}`,
|
|
2617
|
-
tokens: [constructorIdentity, callable],
|
|
2618
|
-
operationToken: callable,
|
|
2619
|
-
addressToken: constructorIdentity,
|
|
2620
|
-
operationIdentityKey: `${path}#${owner}:${callable}@${index + 1}`,
|
|
2621
|
-
});
|
|
2622
|
-
}
|
|
2623
|
-
}
|
|
2624
|
-
});
|
|
2625
|
-
}
|
|
2626
|
-
function productPublicationWorkflowJobs(path, lines) {
|
|
2627
|
-
if (structuredPurposeTokens(basename(path)).some((token) => EXCLUDED_DIRECTORY_PURPOSE_TOKENS.has(token) || token === 'documentation'))
|
|
2628
|
-
return new Set();
|
|
2629
|
-
const jobs = new Map();
|
|
2630
|
-
let jobsIndent = null;
|
|
2631
|
-
let jobIndent = null;
|
|
2632
|
-
let currentJob = null;
|
|
2633
|
-
lines.forEach((line) => {
|
|
2634
|
-
const trimmed = line.trim();
|
|
2635
|
-
const indent = indentation(line);
|
|
2636
|
-
if (/^jobs:\s*(?:#.*)?$/.test(trimmed)) {
|
|
2637
|
-
jobsIndent = indent;
|
|
2638
|
-
jobIndent = null;
|
|
2639
|
-
currentJob = null;
|
|
2640
|
-
return;
|
|
2641
|
-
}
|
|
2642
|
-
if (jobsIndent === null)
|
|
2643
|
-
return;
|
|
2644
|
-
if (trimmed && indent <= jobsIndent) {
|
|
2645
|
-
jobsIndent = null;
|
|
2646
|
-
currentJob = null;
|
|
2647
|
-
return;
|
|
2648
|
-
}
|
|
2649
|
-
const job = line.match(/^\s*([A-Za-z0-9_.-]+):\s*(?:#.*)?$/)?.[1];
|
|
2650
|
-
if (job && indent > jobsIndent && (jobIndent === null || indent === jobIndent)) {
|
|
2651
|
-
jobIndent ??= indent;
|
|
2652
|
-
currentJob = job;
|
|
2653
|
-
jobs.set(job, []);
|
|
2654
|
-
return;
|
|
2655
|
-
}
|
|
2656
|
-
if (currentJob)
|
|
2657
|
-
jobs.get(currentJob)?.push(line);
|
|
2658
|
-
});
|
|
2659
|
-
const publicationOpcode = /(?:\bnpm\s+publish\b|\bmvn(?:w|\s)[^\n]*\bdeploy\b|\bgradlew?\b[^\n]*\bpublish\b|\btwine\s+upload\b|\bdocker\s+push\b|\bgh\s+release\s+create\b|\bcargo\s+publish\b|\bgem\s+push\b|\bgoReleaser\b|uses:\s*[^\s#]*(?:publish|release|deploy|upload-pages-artifact|deploy-pages)[^\s#]*)/i;
|
|
2660
|
-
return new Set([...jobs].filter(([, body]) => publicationOpcode.test(body.join('\n'))).map(([job]) => job));
|
|
2661
|
-
}
|
|
2662
|
-
function isProductPublicationCommand(command) {
|
|
2663
|
-
return /^(?:\.\/)?(?:mvnw?|gradlew?)\b.*\b(?:deploy|publish)\b|^npm\s+publish\b|^twine\s+upload\b|^docker\s+push\b|^gh\s+release\s+create\b|^cargo\s+publish\b|^gem\s+push\b|^git\s+(?:config|add|commit|tag|push)\b/i
|
|
2664
|
-
.test(command.trim());
|
|
2665
|
-
}
|
|
2666
|
-
function scanWorkflowAndRecipeOperations(path, lines, records) {
|
|
2667
|
-
const workflow = path.startsWith('.github/workflows/') || /(?:^|\/)(?:ci|pipeline)s?\.(?:ya?ml)$/i.test(path);
|
|
2668
|
-
if (workflow) {
|
|
2669
|
-
const publicationJobs = productPublicationWorkflowJobs(path, lines);
|
|
2670
|
-
let jobsIndent = null;
|
|
2671
|
-
let jobIndent = null;
|
|
2672
|
-
let currentJob = 'workflow';
|
|
2673
|
-
let triggerIndent = null;
|
|
2674
|
-
let triggerChildIndent = null;
|
|
2675
|
-
let currentTrigger = '';
|
|
2676
|
-
let currentScheduleEmitted = false;
|
|
2677
|
-
let blockRunIndent = null;
|
|
2678
|
-
const emitSecretReads = (line, index) => {
|
|
2679
|
-
for (const secret of line.matchAll(/\$\{\{\s*secrets\.([A-Za-z_][A-Za-z0-9_]*)\s*\}\}/g)) {
|
|
2680
|
-
addRecord(records, {
|
|
2681
|
-
kind: publicationJobs.has(currentJob)
|
|
2682
|
-
? 'workflow-publication-secret-read-operation'
|
|
2683
|
-
: 'workflow-secret-read-operation',
|
|
2684
|
-
projections: ['external_dependencies'],
|
|
2685
|
-
presumptiveInventory: true,
|
|
2686
|
-
path,
|
|
2687
|
-
line: index + 1,
|
|
2688
|
-
anchor: line,
|
|
2689
|
-
owner: `workflow-job:${currentJob}`,
|
|
2690
|
-
identity: `workflow-secret:${currentJob}:${secret[1]}@${index + 1}`,
|
|
2691
|
-
tokens: [currentJob, secret[1]],
|
|
2692
|
-
operationToken: 'read-workflow-secret',
|
|
2693
|
-
addressToken: secret[1],
|
|
2694
|
-
operationIdentityKey: `${path}#jobs.${currentJob}:secrets.${secret[1]}@${index + 1}`,
|
|
2695
|
-
});
|
|
2696
|
-
}
|
|
2697
|
-
};
|
|
2698
|
-
lines.forEach((line, index) => {
|
|
2699
|
-
const trimmed = line.trim();
|
|
2700
|
-
const indent = indentation(line);
|
|
2701
|
-
emitSecretReads(line, index);
|
|
2702
|
-
if (blockRunIndent !== null) {
|
|
2703
|
-
if (trimmed && !trimmed.startsWith('#') && indent > blockRunIndent) {
|
|
2704
|
-
const command = trimmed.replace(/^[-@]+/, '');
|
|
2705
|
-
const installProfileCommand = /\bpip(?:3)?\s+install\b.*(?:-r|--requirement)\b/.test(command);
|
|
2706
|
-
const projections = ['external_dependencies'];
|
|
2707
|
-
if (/^git\s+(?:config|add|commit|checkout|reset|update-ref)\b/.test(command)) {
|
|
2708
|
-
projections.push('stateful_resources');
|
|
2709
|
-
}
|
|
2710
|
-
addRecord(records, {
|
|
2711
|
-
kind: publicationJobs.has(currentJob) && isProductPublicationCommand(command)
|
|
2712
|
-
? 'workflow-publication-command-operation'
|
|
2713
|
-
: 'workflow-command-operation',
|
|
2714
|
-
projections,
|
|
2715
|
-
presumptiveInventory: !installProfileCommand,
|
|
2716
|
-
path,
|
|
2717
|
-
line: index + 1,
|
|
2718
|
-
anchor: line,
|
|
2719
|
-
owner: `workflow-job:${currentJob}`,
|
|
2720
|
-
identity: `workflow-command:${currentJob}:${command}@${index + 1}`,
|
|
2721
|
-
tokens: [currentJob, command],
|
|
2722
|
-
operationToken: command.split(/\s+/)[0],
|
|
2723
|
-
operationIdentityKey: `${path}#jobs.${currentJob}:run@${index + 1}`,
|
|
2724
|
-
});
|
|
2725
|
-
return;
|
|
2726
|
-
}
|
|
2727
|
-
if (trimmed && indent <= blockRunIndent)
|
|
2728
|
-
blockRunIndent = null;
|
|
2729
|
-
}
|
|
2730
|
-
if (/^\s*jobs:\s*(?:#.*)?$/.test(line)) {
|
|
2731
|
-
jobsIndent = indent;
|
|
2732
|
-
jobIndent = null;
|
|
2733
|
-
triggerIndent = null;
|
|
2734
|
-
triggerChildIndent = null;
|
|
2735
|
-
currentTrigger = '';
|
|
2736
|
-
return;
|
|
2737
|
-
}
|
|
2738
|
-
if (jobsIndent !== null && trimmed && indent <= jobsIndent && !/^jobs:/.test(trimmed)) {
|
|
2739
|
-
jobsIndent = null;
|
|
2740
|
-
jobIndent = null;
|
|
2741
|
-
currentJob = 'workflow';
|
|
2742
|
-
}
|
|
2743
|
-
if (jobsIndent !== null) {
|
|
2744
|
-
const job = line.match(/^\s*([A-Za-z0-9_.-]+):\s*(?:#.*)?$/)?.[1];
|
|
2745
|
-
if (job && indent > jobsIndent && (jobIndent === null || indent === jobIndent)) {
|
|
2746
|
-
jobIndent ??= indent;
|
|
2747
|
-
currentJob = job;
|
|
2748
|
-
addRecord(records, {
|
|
2749
|
-
kind: publicationJobs.has(job)
|
|
2750
|
-
? 'workflow-product-publication-runtime-unit'
|
|
2751
|
-
: 'workflow-controlled-runtime-unit',
|
|
2752
|
-
projections: ['runtime_roles'],
|
|
2753
|
-
presumptiveInventory: true,
|
|
2754
|
-
path,
|
|
2755
|
-
line: index + 1,
|
|
2756
|
-
anchor: line,
|
|
2757
|
-
owner: 'workflow-jobs',
|
|
2758
|
-
identity: `workflow-job:${path}#jobs.${job}`,
|
|
2759
|
-
tokens: [path, job],
|
|
2760
|
-
selectorKey: job,
|
|
2761
|
-
implementationKey: `${path}#jobs.${job}`,
|
|
2762
|
-
operationToken: 'execute-workflow-job',
|
|
2763
|
-
addressToken: `${path}#jobs.${job}`,
|
|
2764
|
-
operationIdentityKey: `${path}#jobs.${job}`,
|
|
2765
|
-
});
|
|
2766
|
-
}
|
|
2767
|
-
}
|
|
2768
|
-
if (/^\s*on:\s*(?:#.*)?$/.test(line)) {
|
|
2769
|
-
triggerIndent = indent;
|
|
2770
|
-
triggerChildIndent = null;
|
|
2771
|
-
currentTrigger = '';
|
|
2772
|
-
currentScheduleEmitted = false;
|
|
2773
|
-
return;
|
|
2774
|
-
}
|
|
2775
|
-
if (triggerIndent !== null && trimmed && indent <= triggerIndent && !/^on:/.test(trimmed)) {
|
|
2776
|
-
triggerIndent = null;
|
|
2777
|
-
triggerChildIndent = null;
|
|
2778
|
-
currentTrigger = '';
|
|
2779
|
-
}
|
|
2780
|
-
if (triggerIndent !== null && indent > triggerIndent) {
|
|
2781
|
-
const trigger = line.match(/^\s*([A-Za-z0-9_.-]+):(?:\s*.*)?$/)?.[1];
|
|
2782
|
-
if (trigger && (triggerChildIndent === null || indent === triggerChildIndent)) {
|
|
2783
|
-
triggerChildIndent ??= indent;
|
|
2784
|
-
currentTrigger = trigger;
|
|
2785
|
-
currentScheduleEmitted = false;
|
|
2786
|
-
if (trigger !== 'schedule')
|
|
2787
|
-
addRecord(records, {
|
|
2788
|
-
kind: publicationJobs.size > 0
|
|
2789
|
-
? 'workflow-publication-trigger-operation'
|
|
2790
|
-
: 'workflow-trigger-operation',
|
|
2791
|
-
projections: ['external_dependencies'],
|
|
2792
|
-
presumptiveInventory: true,
|
|
2793
|
-
path,
|
|
2794
|
-
line: index + 1,
|
|
2795
|
-
anchor: line,
|
|
2796
|
-
owner: 'workflow-trigger',
|
|
2797
|
-
identity: `workflow-trigger:${path}:${trigger}@${index + 1}`,
|
|
2798
|
-
tokens: [trigger],
|
|
2799
|
-
operationToken: trigger,
|
|
2800
|
-
addressToken: `${path}#on.${trigger}`,
|
|
2801
|
-
operationIdentityKey: `${path}#on.${trigger}@${index + 1}`,
|
|
2802
|
-
});
|
|
2803
|
-
}
|
|
2804
|
-
const cron = line.match(/^\s*-\s*cron:\s*(['"]?)(.+?)\1\s*(?:#.*)?$/)?.[2];
|
|
2805
|
-
if (currentTrigger === 'schedule' && cron) {
|
|
2806
|
-
currentScheduleEmitted = true;
|
|
2807
|
-
addRecord(records, {
|
|
2808
|
-
kind: publicationJobs.size > 0
|
|
2809
|
-
? 'workflow-publication-trigger-operation'
|
|
2810
|
-
: 'workflow-trigger-operation',
|
|
2811
|
-
projections: ['external_dependencies'],
|
|
2812
|
-
presumptiveInventory: true,
|
|
2813
|
-
path,
|
|
2814
|
-
line: index + 1,
|
|
2815
|
-
anchor: line,
|
|
2816
|
-
owner: 'workflow-trigger',
|
|
2817
|
-
identity: `workflow-trigger:${path}:schedule:${cron}@${index + 1}`,
|
|
2818
|
-
tokens: ['schedule', cron],
|
|
2819
|
-
operationToken: 'schedule',
|
|
2820
|
-
addressToken: cron,
|
|
2821
|
-
operationIdentityKey: `${path}#on.schedule@${index + 1}`,
|
|
2822
|
-
});
|
|
2823
|
-
}
|
|
2824
|
-
if (currentTrigger === 'schedule' &&
|
|
2825
|
-
!currentScheduleEmitted &&
|
|
2826
|
-
trigger &&
|
|
2827
|
-
trigger !== 'schedule')
|
|
2828
|
-
currentScheduleEmitted = true;
|
|
2829
|
-
}
|
|
2830
|
-
const uses = line.match(/^\s*-?\s*uses:\s*([^\s#]+)/)?.[1];
|
|
2831
|
-
if (uses) {
|
|
2832
|
-
addRecord(records, {
|
|
2833
|
-
kind: publicationJobs.has(currentJob)
|
|
2834
|
-
? 'workflow-publication-action-operation'
|
|
2835
|
-
: 'workflow-action-operation',
|
|
2836
|
-
projections: ['external_dependencies'], presumptiveInventory: true,
|
|
2837
|
-
path, line: index + 1, anchor: line, owner: `workflow-job:${currentJob}`,
|
|
2838
|
-
identity: `workflow-action:${currentJob}:${uses}@${index + 1}`, tokens: [currentJob, uses],
|
|
2839
|
-
operationToken: 'uses', addressToken: uses,
|
|
2840
|
-
operationIdentityKey: `${path}#jobs.${currentJob}:uses@${index + 1}`,
|
|
2841
|
-
});
|
|
2842
|
-
if (publicationJobs.has(currentJob) && /(?:cache|artifact|publish|release|deploy)/i.test(uses)) {
|
|
2843
|
-
addRecord(records, {
|
|
2844
|
-
kind: 'workflow-publication-stage-state',
|
|
2845
|
-
projections: ['stateful_resources'],
|
|
2846
|
-
presumptiveInventory: true,
|
|
2847
|
-
path,
|
|
2848
|
-
line: index + 1,
|
|
2849
|
-
anchor: line,
|
|
2850
|
-
owner: `workflow-job:${currentJob}`,
|
|
2851
|
-
identity: `publication-stage:${currentJob}:${uses}@${index + 1}`,
|
|
2852
|
-
tokens: [currentJob, uses],
|
|
2853
|
-
addressToken: `${path}#jobs.${currentJob}:uses@${index + 1}`,
|
|
2854
|
-
materializationRootIdentityKey: `${path}#jobs.${currentJob}`,
|
|
2855
|
-
reproductionOperationToken: `uses:${uses}`,
|
|
2856
|
-
});
|
|
2857
|
-
}
|
|
2858
|
-
}
|
|
2859
|
-
const run = line.match(/^\s*-?\s*run:\s*(.+)$/)?.[1];
|
|
2860
|
-
if (run === '|' || run === '>') {
|
|
2861
|
-
blockRunIndent = indent;
|
|
2862
|
-
return;
|
|
2863
|
-
}
|
|
2864
|
-
if (run && run !== '|' && run !== '>')
|
|
2865
|
-
addRecord(records, {
|
|
2866
|
-
kind: publicationJobs.has(currentJob) && isProductPublicationCommand(run)
|
|
2867
|
-
? 'workflow-publication-command-operation'
|
|
2868
|
-
: 'workflow-command-operation',
|
|
2869
|
-
projections: ['external_dependencies'], presumptiveInventory: true,
|
|
2870
|
-
path, line: index + 1, anchor: line, owner: `workflow-job:${currentJob}`,
|
|
2871
|
-
identity: `workflow-command:${currentJob}:${run}@${index + 1}`, tokens: [currentJob, run],
|
|
2872
|
-
operationToken: run.split(/\s+/)[0],
|
|
2873
|
-
operationIdentityKey: `${path}#jobs.${currentJob}:run@${index + 1}`,
|
|
2874
|
-
});
|
|
2875
|
-
const runner = line.match(/^\s*runs-on:\s*([^#]+?)\s*$/)?.[1];
|
|
2876
|
-
if (runner)
|
|
2877
|
-
addRecord(records, {
|
|
2878
|
-
kind: publicationJobs.has(currentJob)
|
|
2879
|
-
? 'workflow-publication-host-allocation-operation'
|
|
2880
|
-
: 'workflow-host-allocation-operation',
|
|
2881
|
-
projections: ['external_dependencies'],
|
|
2882
|
-
presumptiveInventory: true,
|
|
2883
|
-
path,
|
|
2884
|
-
line: index + 1,
|
|
2885
|
-
anchor: line,
|
|
2886
|
-
owner: `workflow-job:${currentJob}`,
|
|
2887
|
-
identity: `workflow-host:${currentJob}:${runner}@${index + 1}`,
|
|
2888
|
-
tokens: [currentJob, runner],
|
|
2889
|
-
operationToken: 'allocate-workflow-host',
|
|
2890
|
-
addressToken: runner,
|
|
2891
|
-
operationIdentityKey: `${path}#jobs.${currentJob}:runs-on@${index + 1}`,
|
|
2892
|
-
});
|
|
2893
|
-
});
|
|
2894
|
-
}
|
|
2895
|
-
if (basename(path).toLowerCase() === 'makefile') {
|
|
2896
|
-
lines.forEach((line, index) => {
|
|
2897
|
-
if (!/^\t\S/.test(line) || /^\t[@-]?(?:#|echo\b)/.test(line))
|
|
2898
|
-
return;
|
|
2899
|
-
const command = line.trim().replace(/^[@-]+/, '');
|
|
2900
|
-
const projections = ['external_dependencies'];
|
|
2901
|
-
if (/^(?:cp|install|mkdir|mv|rsync|sed|stubgen|tar|touch|uv\s+build)\b/.test(command)) {
|
|
2902
|
-
projections.push('stateful_resources');
|
|
2903
|
-
}
|
|
2904
|
-
addRecord(records, {
|
|
2905
|
-
kind: 'build-command-operation', projections, presumptiveInventory: true,
|
|
2906
|
-
path, line: index + 1, anchor: line, owner: 'make-recipe',
|
|
2907
|
-
identity: `make-command:${command}@${index + 1}`, tokens: [command], operationToken: command.split(/\s+/)[0],
|
|
2908
|
-
});
|
|
2909
|
-
});
|
|
2910
|
-
}
|
|
2911
|
-
}
|
|
2912
|
-
const NON_PRODUCTION_OPTIONAL_GROUP_PARTS = new Set([
|
|
2913
|
-
'bench',
|
|
2914
|
-
'benchmark',
|
|
2915
|
-
'benchmarks',
|
|
2916
|
-
'benchmarking',
|
|
2917
|
-
'ci',
|
|
2918
|
-
'dev',
|
|
2919
|
-
'development',
|
|
2920
|
-
'doc',
|
|
2921
|
-
'docs',
|
|
2922
|
-
'documentation',
|
|
2923
|
-
'lint',
|
|
2924
|
-
'linting',
|
|
2925
|
-
'qa',
|
|
2926
|
-
'release',
|
|
2927
|
-
'test',
|
|
2928
|
-
'tests',
|
|
2929
|
-
'testing',
|
|
2930
|
-
'type',
|
|
2931
|
-
'types',
|
|
2932
|
-
'typing',
|
|
2933
|
-
]);
|
|
2934
|
-
function executedCommandSurfaces(path, lines) {
|
|
2935
|
-
const lowerName = basename(path).toLowerCase();
|
|
2936
|
-
const extension = extname(lowerName);
|
|
2937
|
-
const workflow = path.startsWith('.github/workflows/') ||
|
|
2938
|
-
/(?:^|\/)(?:ci|pipeline)s?\.(?:ya?ml)$/i.test(path);
|
|
2939
|
-
const surfaces = [];
|
|
2940
|
-
let blockRunIndent = null;
|
|
2941
|
-
lines.forEach((line, index) => {
|
|
2942
|
-
const trimmed = line.trim();
|
|
2943
|
-
const indent = indentation(line);
|
|
2944
|
-
if (workflow) {
|
|
2945
|
-
if (blockRunIndent !== null) {
|
|
2946
|
-
if (trimmed && !trimmed.startsWith('#') && indent > blockRunIndent) {
|
|
2947
|
-
surfaces.push({
|
|
2948
|
-
line: index + 1,
|
|
2949
|
-
anchor: line,
|
|
2950
|
-
command: trimmed,
|
|
2951
|
-
preserveWholeCommand: true,
|
|
2952
|
-
});
|
|
2953
|
-
return;
|
|
2954
|
-
}
|
|
2955
|
-
if (trimmed && indent <= blockRunIndent)
|
|
2956
|
-
blockRunIndent = null;
|
|
2957
|
-
}
|
|
2958
|
-
const run = line.match(/^\s*-?\s*run:\s*(.+)$/)?.[1];
|
|
2959
|
-
if (run === '|' || run === '>') {
|
|
2960
|
-
blockRunIndent = indent;
|
|
2961
|
-
return;
|
|
2962
|
-
}
|
|
2963
|
-
if (run)
|
|
2964
|
-
surfaces.push({
|
|
2965
|
-
line: index + 1,
|
|
2966
|
-
anchor: line,
|
|
2967
|
-
command: run,
|
|
2968
|
-
preserveWholeCommand: true,
|
|
2969
|
-
});
|
|
2970
|
-
return;
|
|
2971
|
-
}
|
|
2972
|
-
if (lowerName === 'makefile') {
|
|
2973
|
-
if (/^\t[@-]?(?!#|echo\b)\S/.test(line)) {
|
|
2974
|
-
surfaces.push({
|
|
2975
|
-
line: index + 1,
|
|
2976
|
-
anchor: line,
|
|
2977
|
-
command: trimmed.replace(/^[@-]+/, ''),
|
|
2978
|
-
preserveWholeCommand: true,
|
|
2979
|
-
});
|
|
2980
|
-
}
|
|
2981
|
-
return;
|
|
2982
|
-
}
|
|
2983
|
-
if (lowerName === 'dockerfile') {
|
|
2984
|
-
const run = line.match(/^\s*RUN\s+(.+)$/i)?.[1];
|
|
2985
|
-
if (run)
|
|
2986
|
-
surfaces.push({
|
|
2987
|
-
line: index + 1,
|
|
2988
|
-
anchor: line,
|
|
2989
|
-
command: run,
|
|
2990
|
-
preserveWholeCommand: true,
|
|
2991
|
-
});
|
|
2992
|
-
return;
|
|
2993
|
-
}
|
|
2994
|
-
if (['.sh', '.bash', '.zsh', '.fish', '.ps1'].includes(extension)) {
|
|
2995
|
-
if (trimmed && !trimmed.startsWith('#')) {
|
|
2996
|
-
surfaces.push({
|
|
2997
|
-
line: index + 1,
|
|
2998
|
-
anchor: line,
|
|
2999
|
-
command: trimmed,
|
|
3000
|
-
preserveWholeCommand: true,
|
|
3001
|
-
});
|
|
3002
|
-
}
|
|
3003
|
-
return;
|
|
3004
|
-
}
|
|
3005
|
-
if (/\b(?:subprocess\.(?:run|Popen|call|check_call|check_output)|os\.system|child_process\.(?:spawn|exec|execFile)|run_command|exec_command)\s*\(/.test(line))
|
|
3006
|
-
surfaces.push({
|
|
3007
|
-
line: index + 1,
|
|
3008
|
-
anchor: line,
|
|
3009
|
-
command: line,
|
|
3010
|
-
preserveWholeCommand: false,
|
|
3011
|
-
});
|
|
3012
|
-
});
|
|
3013
|
-
return surfaces;
|
|
3014
|
-
}
|
|
3015
|
-
function discoverPythonRequirementProfiles(root, tracked) {
|
|
3016
|
-
const trackedSet = new Set(tracked);
|
|
3017
|
-
const profiles = new Map();
|
|
3018
|
-
for (const path of tracked) {
|
|
3019
|
-
if (/^requirements(?:[._-](?:prod|production|runtime))?\.txt$/i.test(basename(path))) {
|
|
3020
|
-
profiles.set(path, { path, installReferences: [] });
|
|
3021
|
-
}
|
|
3022
|
-
}
|
|
3023
|
-
for (const sourcePath of tracked) {
|
|
3024
|
-
if (!shouldScan(sourcePath))
|
|
3025
|
-
continue;
|
|
3026
|
-
let lines = null;
|
|
3027
|
-
try {
|
|
3028
|
-
lines = textLines(root, sourcePath);
|
|
3029
|
-
}
|
|
3030
|
-
catch {
|
|
3031
|
-
lines = null;
|
|
3032
|
-
}
|
|
3033
|
-
if (!lines)
|
|
3034
|
-
continue;
|
|
3035
|
-
for (const surface of executedCommandSurfaces(sourcePath, lines)) {
|
|
3036
|
-
for (const match of surface.command.matchAll(/\b(?:python(?:3(?:\.\d+)?)?\s+-m\s+)?pip(?:3)?\s+install\b[^;|&\n]*?(?:-r|--requirement)(?:\s+|=)(?:['"])([^'"]+)['"]/g)) {
|
|
3037
|
-
const rawPath = (match[1] ?? '').trim();
|
|
3038
|
-
const normalized = normalizePath(rawPath).replace(/^\.\//, '');
|
|
3039
|
-
if (!normalized || !trackedSet.has(normalized))
|
|
3040
|
-
continue;
|
|
3041
|
-
const profile = profiles.get(normalized) ?? {
|
|
3042
|
-
path: normalized,
|
|
3043
|
-
installReferences: [],
|
|
3044
|
-
};
|
|
3045
|
-
const command = surface.preserveWholeCommand
|
|
3046
|
-
? surface.command.trim()
|
|
3047
|
-
: match[0].trim();
|
|
3048
|
-
if (!profile.installReferences.some((reference) => reference.path === sourcePath && reference.line === surface.line && reference.command === command))
|
|
3049
|
-
profile.installReferences.push({
|
|
3050
|
-
path: sourcePath,
|
|
3051
|
-
line: surface.line,
|
|
3052
|
-
anchor: surface.anchor,
|
|
3053
|
-
command,
|
|
3054
|
-
});
|
|
3055
|
-
profiles.set(normalized, profile);
|
|
3056
|
-
}
|
|
3057
|
-
for (const match of surface.command.matchAll(/\b(?:python(?:3(?:\.\d+)?)?\s+-m\s+)?pip(?:3)?\s+install\b[^;|&\n]*?(?:-r|--requirement)(?:\s+|=)([^\s;'"|&]+)/g)) {
|
|
3058
|
-
const normalized = normalizePath(match[1]).replace(/^\.\//, '');
|
|
3059
|
-
if (!normalized || !trackedSet.has(normalized))
|
|
3060
|
-
continue;
|
|
3061
|
-
const profile = profiles.get(normalized) ?? {
|
|
3062
|
-
path: normalized,
|
|
3063
|
-
installReferences: [],
|
|
3064
|
-
};
|
|
3065
|
-
const command = surface.preserveWholeCommand
|
|
3066
|
-
? surface.command.trim()
|
|
3067
|
-
: match[0].trim();
|
|
3068
|
-
if (!profile.installReferences.some((reference) => reference.path === sourcePath && reference.line === surface.line && reference.command === command))
|
|
3069
|
-
profile.installReferences.push({
|
|
3070
|
-
path: sourcePath,
|
|
3071
|
-
line: surface.line,
|
|
3072
|
-
anchor: surface.anchor,
|
|
3073
|
-
command,
|
|
3074
|
-
});
|
|
3075
|
-
profiles.set(normalized, profile);
|
|
3076
|
-
}
|
|
3077
|
-
}
|
|
3078
|
-
}
|
|
3079
|
-
for (const profile of profiles.values()) {
|
|
3080
|
-
profile.installReferences.sort((left, right) => left.path.localeCompare(right.path) || left.line - right.line || left.command.localeCompare(right.command));
|
|
3081
|
-
}
|
|
3082
|
-
return profiles;
|
|
3083
|
-
}
|
|
3084
|
-
function scanPythonRequirementProfile(profile, lines, records) {
|
|
3085
|
-
const requirements = lines.flatMap((line, index) => {
|
|
3086
|
-
const trimmed = line.trim();
|
|
3087
|
-
if (!trimmed || trimmed.startsWith('#') || /^-/.test(trimmed))
|
|
3088
|
-
return [];
|
|
3089
|
-
const requirement = trimmed.match(/^([A-Za-z0-9_.-]+)(\[[^\]]+\])?([^;#\s]*)(?:\s*;[^#]*)?(?:\s*#.*)?$/);
|
|
3090
|
-
if (!requirement)
|
|
3091
|
-
return [];
|
|
3092
|
-
return [{
|
|
3093
|
-
line: index + 1,
|
|
3094
|
-
anchor: line,
|
|
3095
|
-
name: normalizePackageName(requirement[1]),
|
|
3096
|
-
extras: requirement[2] ?? '',
|
|
3097
|
-
version: requirement[3] || 'unversioned',
|
|
3098
|
-
raw: `${requirement[1]}${requirement[2] ?? ''}${requirement[3] ?? ''}`,
|
|
3099
|
-
}];
|
|
3100
|
-
});
|
|
3101
|
-
const reproductionOperationToken = profile.installReferences[0]?.command;
|
|
3102
|
-
addRecord(records, {
|
|
3103
|
-
kind: 'python-requirements-installation-profile-state',
|
|
3104
|
-
projections: ['stateful_resources'],
|
|
3105
|
-
presumptiveInventory: true,
|
|
3106
|
-
path: profile.path,
|
|
3107
|
-
line: 1,
|
|
3108
|
-
anchor: lines[0] ?? profile.path,
|
|
3109
|
-
owner: 'python-requirements-profile',
|
|
3110
|
-
identity: `requirements-profile:${profile.path}`,
|
|
3111
|
-
tokens: [profile.path, ...requirements.map((requirement) => requirement.raw)],
|
|
3112
|
-
addressToken: profile.path,
|
|
3113
|
-
materializationRootIdentityKey: profile.path,
|
|
3114
|
-
reproductionOperationToken,
|
|
3115
|
-
});
|
|
3116
|
-
for (const requirement of requirements)
|
|
3117
|
-
addRecord(records, {
|
|
3118
|
-
kind: 'python-requirement-variant-state',
|
|
3119
|
-
projections: ['stateful_resources'],
|
|
3120
|
-
presumptiveInventory: true,
|
|
3121
|
-
path: profile.path,
|
|
3122
|
-
line: requirement.line,
|
|
3123
|
-
anchor: requirement.anchor,
|
|
3124
|
-
owner: `python-requirements-profile:${profile.path}`,
|
|
3125
|
-
identity: `requirement:${profile.path}:${requirement.line}:${requirement.raw}`,
|
|
3126
|
-
tokens: [profile.path, requirement.raw, requirement.name, requirement.extras, requirement.version],
|
|
3127
|
-
addressToken: requirement.name,
|
|
3128
|
-
packageName: requirement.name,
|
|
3129
|
-
packageVersion: requirement.version,
|
|
3130
|
-
materializationRootIdentityKey: profile.path,
|
|
3131
|
-
reproductionOperationToken,
|
|
3132
|
-
});
|
|
3133
|
-
for (const reference of profile.installReferences)
|
|
3134
|
-
addRecord(records, {
|
|
3135
|
-
kind: 'python-requirements-profile-install-operation',
|
|
3136
|
-
projections: ['external_dependencies'],
|
|
3137
|
-
presumptiveInventory: true,
|
|
3138
|
-
path: reference.path,
|
|
3139
|
-
line: reference.line,
|
|
3140
|
-
anchor: reference.anchor,
|
|
3141
|
-
owner: 'executed-install-command',
|
|
3142
|
-
identity: `install-profile:${profile.path}:${reference.path}:${reference.line}:${reference.command}`,
|
|
3143
|
-
tokens: [profile.path, reference.command],
|
|
3144
|
-
operationToken: reference.command,
|
|
3145
|
-
addressToken: profile.path,
|
|
3146
|
-
operationIdentityKey: `${reference.path}#install:${reference.line}:${reference.command}`,
|
|
3147
|
-
materializationRootIdentityKey: profile.path,
|
|
3148
|
-
reproductionOperationToken: reference.command,
|
|
3149
|
-
});
|
|
3150
|
-
}
|
|
3151
|
-
function isNonProductionOptionalGroup(value) {
|
|
3152
|
-
return value
|
|
3153
|
-
.trim()
|
|
3154
|
-
.toLowerCase()
|
|
3155
|
-
.split(/[._-]+/)
|
|
3156
|
-
.some((part) => NON_PRODUCTION_OPTIONAL_GROUP_PARTS.has(part));
|
|
3157
|
-
}
|
|
3158
|
-
function optionalGroupName(line) {
|
|
3159
|
-
const match = line.trim().match(/^(?:"([^"]+)"|'([^']+)'|([A-Za-z0-9_.-]+))\s*=\s*\[/);
|
|
3160
|
-
return match?.[1] ?? match?.[2] ?? match?.[3];
|
|
3161
|
-
}
|
|
3162
|
-
function artifactFilename(url) {
|
|
3163
|
-
const path = url.split(/[?#]/, 1)[0];
|
|
3164
|
-
const encoded = path.slice(path.lastIndexOf('/') + 1);
|
|
3165
|
-
try {
|
|
3166
|
-
return decodeURIComponent(encoded);
|
|
3167
|
-
}
|
|
3168
|
-
catch {
|
|
3169
|
-
return encoded;
|
|
3170
|
-
}
|
|
3171
|
-
}
|
|
3172
|
-
function parseUvPackages(lines) {
|
|
3173
|
-
const starts = [];
|
|
3174
|
-
lines.forEach((line, index) => { if (line.trim() === '[[package]]')
|
|
3175
|
-
starts.push(index); });
|
|
3176
|
-
return starts.flatMap((start, position) => {
|
|
3177
|
-
const end = starts[position + 1] ?? lines.length;
|
|
3178
|
-
const block = lines.slice(start, end);
|
|
3179
|
-
const nameIndex = block.findIndex((line) => /^name\s*=\s*"/.test(line));
|
|
3180
|
-
if (nameIndex < 0)
|
|
3181
|
-
return [];
|
|
3182
|
-
const name = block[nameIndex].match(/^name\s*=\s*"([^"]+)"/)?.[1];
|
|
3183
|
-
if (!name)
|
|
3184
|
-
return [];
|
|
3185
|
-
const versionIndex = block.findIndex((line) => /^version\s*=\s*"/.test(line));
|
|
3186
|
-
const version = versionIndex < 0 ? 'workspace' : block[versionIndex].match(/^version\s*=\s*"([^"]+)"/)?.[1] ?? 'workspace';
|
|
3187
|
-
const sourceIndex = block.findIndex((line) => /^source\s*=/.test(line));
|
|
3188
|
-
const sourceLine = sourceIndex < 0 ? block[nameIndex] : block[sourceIndex];
|
|
3189
|
-
const dependencies = new Set();
|
|
3190
|
-
const optionalDependencies = new Set();
|
|
3191
|
-
const artifacts = [];
|
|
3192
|
-
let section = '';
|
|
3193
|
-
let optionalGroup = '';
|
|
3194
|
-
for (let blockIndex = 0; blockIndex < block.length; blockIndex += 1) {
|
|
3195
|
-
const line = block[blockIndex];
|
|
3196
|
-
const trimmed = line.trim();
|
|
3197
|
-
if (/^dependencies\s*=\s*\[/.test(trimmed)) {
|
|
3198
|
-
section = 'dependencies';
|
|
3199
|
-
optionalGroup = '';
|
|
3200
|
-
}
|
|
3201
|
-
else if (trimmed === '[package.optional-dependencies]') {
|
|
3202
|
-
section = 'optional';
|
|
3203
|
-
optionalGroup = '';
|
|
3204
|
-
}
|
|
3205
|
-
else if (trimmed.startsWith('[package.dev-dependencies]') || trimmed.startsWith('[package.metadata]')) {
|
|
3206
|
-
section = 'excluded';
|
|
3207
|
-
optionalGroup = '';
|
|
3208
|
-
}
|
|
3209
|
-
else if (section === 'optional') {
|
|
3210
|
-
const group = optionalGroupName(trimmed);
|
|
3211
|
-
if (group)
|
|
3212
|
-
optionalGroup = group;
|
|
3213
|
-
}
|
|
3214
|
-
for (const dependency of line.matchAll(/\{\s*name\s*=\s*"([^"]+)"/g)) {
|
|
3215
|
-
if (section === 'dependencies')
|
|
3216
|
-
dependencies.add(dependency[1]);
|
|
3217
|
-
if (section === 'optional' &&
|
|
3218
|
-
optionalGroup &&
|
|
3219
|
-
!isNonProductionOptionalGroup(optionalGroup))
|
|
3220
|
-
optionalDependencies.add(dependency[1]);
|
|
3221
|
-
}
|
|
3222
|
-
for (const artifact of line.matchAll(/\{[^{}]*\burl\s*=\s*"([^"]+)"[^{}]*\bhash\s*=\s*"(sha256:[0-9a-fA-F]+)"[^{}]*\}/g)) {
|
|
3223
|
-
const url = artifact[1];
|
|
3224
|
-
const filename = artifactFilename(url);
|
|
3225
|
-
artifacts.push({
|
|
3226
|
-
kind: /^sdist\s*=/.test(trimmed) || !filename.endsWith('.whl') ? 'sdist' : 'wheel',
|
|
3227
|
-
filename,
|
|
3228
|
-
url,
|
|
3229
|
-
hash: artifact[2].toLowerCase(),
|
|
3230
|
-
line: start + blockIndex + 1,
|
|
3231
|
-
anchor: line,
|
|
3232
|
-
});
|
|
3233
|
-
}
|
|
3234
|
-
}
|
|
3235
|
-
return [{
|
|
3236
|
-
name,
|
|
3237
|
-
version,
|
|
3238
|
-
line: start + nameIndex + 1,
|
|
3239
|
-
sourceLine,
|
|
3240
|
-
editable: /editable\s*=|virtual\s*=/.test(sourceLine),
|
|
3241
|
-
dependencies,
|
|
3242
|
-
optionalDependencies,
|
|
3243
|
-
artifacts,
|
|
3244
|
-
}];
|
|
3245
|
-
});
|
|
3246
|
-
}
|
|
3247
|
-
function scanUvLock(path, lines, records) {
|
|
3248
|
-
const packages = parseUvPackages(lines);
|
|
3249
|
-
const byName = new Map();
|
|
3250
|
-
for (const pkg of packages) {
|
|
3251
|
-
const variants = byName.get(pkg.name) ?? [];
|
|
3252
|
-
variants.push(pkg);
|
|
3253
|
-
byName.set(pkg.name, variants);
|
|
3254
|
-
}
|
|
3255
|
-
const roots = packages.filter((pkg) => pkg.editable);
|
|
3256
|
-
const rootNames = new Set(roots.map((pkg) => pkg.name));
|
|
3257
|
-
const queue = roots.flatMap((pkg) => [...pkg.dependencies, ...pkg.optionalDependencies]);
|
|
3258
|
-
const reachable = new Set();
|
|
3259
|
-
while (queue.length > 0) {
|
|
3260
|
-
const name = queue.shift();
|
|
3261
|
-
if (rootNames.has(name))
|
|
3262
|
-
continue;
|
|
3263
|
-
if (reachable.has(name))
|
|
3264
|
-
continue;
|
|
3265
|
-
const variants = byName.get(name);
|
|
3266
|
-
if (!variants)
|
|
3267
|
-
continue;
|
|
3268
|
-
reachable.add(name);
|
|
3269
|
-
for (const pkg of variants)
|
|
3270
|
-
queue.push(...pkg.dependencies);
|
|
3271
|
-
}
|
|
3272
|
-
for (const name of [...reachable].sort()) {
|
|
3273
|
-
for (const pkg of byName.get(name)) {
|
|
3274
|
-
if (pkg.artifacts.length > 0) {
|
|
3275
|
-
for (const artifact of pkg.artifacts) {
|
|
3276
|
-
const tokens = [
|
|
3277
|
-
name,
|
|
3278
|
-
pkg.version,
|
|
3279
|
-
artifact.kind,
|
|
3280
|
-
artifact.filename,
|
|
3281
|
-
artifact.url,
|
|
3282
|
-
artifact.hash,
|
|
3283
|
-
];
|
|
3284
|
-
const variantIdentity = `${name}@${pkg.version}:${artifact.kind}:${artifact.hash}:${artifact.url}`;
|
|
3285
|
-
addRecord(records, {
|
|
3286
|
-
kind: 'locked-production-package-artifact-state',
|
|
3287
|
-
projections: ['stateful_resources'],
|
|
3288
|
-
presumptiveInventory: true,
|
|
3289
|
-
path,
|
|
3290
|
-
line: artifact.line,
|
|
3291
|
-
anchor: artifact.anchor,
|
|
3292
|
-
owner: 'production-package-artifact-closure',
|
|
3293
|
-
identity: `package-artifact:${variantIdentity}`,
|
|
3294
|
-
tokens,
|
|
3295
|
-
addressToken: artifact.filename,
|
|
3296
|
-
packageName: name,
|
|
3297
|
-
packageVersion: pkg.version,
|
|
3298
|
-
packageArtifactKind: artifact.kind,
|
|
3299
|
-
packageArtifactFilename: artifact.filename,
|
|
3300
|
-
packageArtifactUrl: artifact.url,
|
|
3301
|
-
packageArtifactHash: artifact.hash,
|
|
3302
|
-
});
|
|
3303
|
-
addRecord(records, {
|
|
3304
|
-
kind: 'locked-production-package-artifact-install-operation',
|
|
3305
|
-
projections: ['external_dependencies'],
|
|
3306
|
-
presumptiveInventory: true,
|
|
3307
|
-
path,
|
|
3308
|
-
line: artifact.line,
|
|
3309
|
-
anchor: artifact.anchor,
|
|
3310
|
-
owner: 'production-package-artifact-closure',
|
|
3311
|
-
identity: `pull-package-artifact:${variantIdentity}`,
|
|
3312
|
-
tokens,
|
|
3313
|
-
operationToken: 'pull-locked-package-artifact',
|
|
3314
|
-
addressToken: artifact.url,
|
|
3315
|
-
packageName: name,
|
|
3316
|
-
packageVersion: pkg.version,
|
|
3317
|
-
packageArtifactKind: artifact.kind,
|
|
3318
|
-
packageArtifactFilename: artifact.filename,
|
|
3319
|
-
packageArtifactUrl: artifact.url,
|
|
3320
|
-
packageArtifactHash: artifact.hash,
|
|
3321
|
-
});
|
|
3322
|
-
}
|
|
3323
|
-
continue;
|
|
3324
|
-
}
|
|
3325
|
-
const anchor = lines[pkg.line - 1];
|
|
3326
|
-
addRecord(records, {
|
|
3327
|
-
kind: 'locked-production-package-state',
|
|
3328
|
-
projections: ['stateful_resources'],
|
|
3329
|
-
presumptiveInventory: true,
|
|
3330
|
-
path, line: pkg.line, anchor, owner: 'production-package-closure',
|
|
3331
|
-
identity: `package:${name}@${pkg.version}`, tokens: [name, pkg.version],
|
|
3332
|
-
addressToken: name, packageName: name, packageVersion: pkg.version,
|
|
3333
|
-
});
|
|
3334
|
-
addRecord(records, {
|
|
3335
|
-
kind: 'locked-production-package-install-operation',
|
|
3336
|
-
projections: ['external_dependencies'],
|
|
3337
|
-
presumptiveInventory: true,
|
|
3338
|
-
path, line: pkg.line, anchor, owner: 'production-package-closure',
|
|
3339
|
-
identity: `install:${name}@${pkg.version}`, tokens: [name, pkg.version],
|
|
3340
|
-
operationToken: 'install-locked-package', packageName: name, packageVersion: pkg.version,
|
|
3341
|
-
});
|
|
3342
|
-
}
|
|
3343
|
-
}
|
|
3344
|
-
}
|
|
3345
|
-
function scanPackageLock(path, text, records) {
|
|
3346
|
-
let parsed;
|
|
3347
|
-
try {
|
|
3348
|
-
parsed = JSON.parse(text);
|
|
3349
|
-
}
|
|
3350
|
-
catch {
|
|
3351
|
-
return;
|
|
3352
|
-
}
|
|
3353
|
-
if (!parsed || typeof parsed !== 'object')
|
|
3354
|
-
return;
|
|
3355
|
-
const lock = parsed;
|
|
3356
|
-
const lines = text.split(/\r?\n/);
|
|
3357
|
-
const emittedLocations = new Set();
|
|
3358
|
-
const emitPackage = (location, packageName, pkg) => {
|
|
3359
|
-
if (!location || pkg.dev === true || emittedLocations.has(location))
|
|
3360
|
-
return;
|
|
3361
|
-
emittedLocations.add(location);
|
|
3362
|
-
const name = pkg.name ?? packageName;
|
|
3363
|
-
const version = pkg.version ?? 'locked';
|
|
3364
|
-
const quotedLocation = JSON.stringify(location);
|
|
3365
|
-
const exactLocatorTokens = [
|
|
3366
|
-
`${quotedLocation}:`,
|
|
3367
|
-
pkg.resolved ? JSON.stringify(pkg.resolved) : '',
|
|
3368
|
-
pkg.integrity ? JSON.stringify(pkg.integrity) : '',
|
|
3369
|
-
JSON.stringify(packageName),
|
|
3370
|
-
].filter(Boolean);
|
|
3371
|
-
const lineIndex = exactLocatorTokens
|
|
3372
|
-
.map((token) => lines.findIndex((line) => line.includes(token)))
|
|
3373
|
-
.find((index) => index >= 0) ?? -1;
|
|
3374
|
-
const line = lineIndex >= 0 ? lineIndex + 1 : 1;
|
|
3375
|
-
const anchor = lines[line - 1] ?? '{';
|
|
3376
|
-
const exactArtifact = Boolean(pkg.resolved && pkg.integrity);
|
|
3377
|
-
const filename = pkg.resolved ? artifactFilename(pkg.resolved) : undefined;
|
|
3378
|
-
for (const [kind, projection, operationToken] of [
|
|
3379
|
-
[exactArtifact ? 'locked-production-package-artifact-state' : 'locked-production-package-state', 'stateful_resources', undefined],
|
|
3380
|
-
[exactArtifact ? 'locked-production-package-artifact-install-operation' : 'locked-production-package-install-operation', 'external_dependencies', exactArtifact ? 'pull-locked-package-artifact' : 'install-locked-package'],
|
|
3381
|
-
])
|
|
3382
|
-
addRecord(records, {
|
|
3383
|
-
kind, projections: [projection], presumptiveInventory: true,
|
|
3384
|
-
path, line, anchor, owner: 'production-package-closure',
|
|
3385
|
-
identity: `${operationToken ? 'install' : 'package'}:${name}@${version}:${location}:${pkg.resolved ?? ''}:${pkg.integrity ?? ''}`,
|
|
3386
|
-
tokens: [name, version, location, pkg.resolved ?? '', pkg.integrity ?? ''],
|
|
3387
|
-
packageName: name,
|
|
3388
|
-
packageVersion: version,
|
|
3389
|
-
...(exactArtifact ? {
|
|
3390
|
-
packageArtifactKind: 'tarball',
|
|
3391
|
-
packageArtifactFilename: filename,
|
|
3392
|
-
packageArtifactUrl: pkg.resolved,
|
|
3393
|
-
packageArtifactHash: pkg.integrity,
|
|
3394
|
-
} : {}),
|
|
3395
|
-
...(operationToken ? { operationToken, addressToken: pkg.resolved } : { addressToken: filename ?? name }),
|
|
3396
|
-
});
|
|
3397
|
-
};
|
|
3398
|
-
if (lock.packages) {
|
|
3399
|
-
Object.entries(lock.packages).forEach(([location, pkg]) => {
|
|
3400
|
-
if (!location)
|
|
3401
|
-
return;
|
|
3402
|
-
const name = pkg.name ?? location.split('/node_modules/').at(-1)
|
|
3403
|
-
.replace(/^node_modules\//, '');
|
|
3404
|
-
emitPackage(location, name, pkg);
|
|
3405
|
-
});
|
|
3406
|
-
return;
|
|
3407
|
-
}
|
|
3408
|
-
const walkLegacy = (dependencies, parentLocation = '') => {
|
|
3409
|
-
Object.entries(dependencies).forEach(([name, pkg]) => {
|
|
3410
|
-
if (pkg.dev === true)
|
|
3411
|
-
return;
|
|
3412
|
-
const location = parentLocation
|
|
3413
|
-
? `${parentLocation}/node_modules/${name}`
|
|
3414
|
-
: `node_modules/${name}`;
|
|
3415
|
-
emitPackage(location, name, pkg);
|
|
3416
|
-
if (pkg.dependencies)
|
|
3417
|
-
walkLegacy(pkg.dependencies, location);
|
|
3418
|
-
});
|
|
3419
|
-
};
|
|
3420
|
-
if (lock.dependencies)
|
|
3421
|
-
walkLegacy(lock.dependencies);
|
|
3422
|
-
}
|
|
3423
|
-
function scanPythonBuildRequirements(path, lines, records) {
|
|
3424
|
-
let inBuild = false;
|
|
3425
|
-
let inRequires = false;
|
|
3426
|
-
lines.forEach((line, index) => {
|
|
3427
|
-
const trimmed = line.trim();
|
|
3428
|
-
if (/^\[.+\]$/.test(trimmed)) {
|
|
3429
|
-
inBuild = trimmed === '[build-system]';
|
|
3430
|
-
inRequires = false;
|
|
3431
|
-
return;
|
|
3432
|
-
}
|
|
3433
|
-
if (!inBuild)
|
|
3434
|
-
return;
|
|
3435
|
-
if (/^requires\s*=\s*\[/.test(trimmed))
|
|
3436
|
-
inRequires = true;
|
|
3437
|
-
if (!inRequires)
|
|
3438
|
-
return;
|
|
3439
|
-
for (const match of line.matchAll(/['"]([A-Za-z0-9_.-]+)(?:\[[^\]]+\])?(?:\s*[^'"]*)?['"]/g)) {
|
|
3440
|
-
const name = match[1].toLowerCase().replace(/_/g, '-');
|
|
3441
|
-
addRecord(records, {
|
|
3442
|
-
kind: 'build-system-package-state', projections: ['stateful_resources'], presumptiveInventory: true,
|
|
3443
|
-
path, line: index + 1, anchor: line, owner: 'build-system', identity: `package:${name}`,
|
|
3444
|
-
tokens: [name], addressToken: name, packageName: name,
|
|
3445
|
-
});
|
|
3446
|
-
addRecord(records, {
|
|
3447
|
-
kind: 'build-system-package-install-operation', projections: ['external_dependencies'], presumptiveInventory: true,
|
|
3448
|
-
path, line: index + 1, anchor: line, owner: 'build-system', identity: `install:${name}`,
|
|
3449
|
-
tokens: [name], operationToken: 'install-build-system-package', packageName: name,
|
|
3450
|
-
});
|
|
3451
|
-
}
|
|
3452
|
-
if (trimmed.includes(']'))
|
|
3453
|
-
inRequires = false;
|
|
3454
|
-
});
|
|
3455
|
-
}
|
|
3456
|
-
function normalizePackageName(value) {
|
|
3457
|
-
return value.trim().toLowerCase().replace(/_/g, '-');
|
|
3458
|
-
}
|
|
3459
|
-
function xmlBlocks(source, tag) {
|
|
3460
|
-
const escaped = tag.replace(/[.*+?^${}()|[\]\\]/g, '\\$&');
|
|
3461
|
-
const pattern = new RegExp(`<${escaped}(?:\\s[^>]*)?>([\\s\\S]*?)<\\/${escaped}>`, 'g');
|
|
3462
|
-
const blocks = [];
|
|
3463
|
-
for (const match of source.matchAll(pattern)) {
|
|
3464
|
-
const start = match.index ?? 0;
|
|
3465
|
-
blocks.push({
|
|
3466
|
-
start,
|
|
3467
|
-
end: start + match[0].length,
|
|
3468
|
-
line: source.slice(0, start).split('\n').length,
|
|
3469
|
-
text: match[1],
|
|
3470
|
-
});
|
|
3471
|
-
}
|
|
3472
|
-
return blocks;
|
|
3473
|
-
}
|
|
3474
|
-
function xmlValue(source, tag) {
|
|
3475
|
-
const escaped = tag.replace(/[.*+?^${}()|[\]\\]/g, '\\$&');
|
|
3476
|
-
return source.match(new RegExp(`<${escaped}(?:\\s[^>]*)?>\\s*([^<]+?)\\s*<\\/${escaped}>`))?.[1].trim();
|
|
3477
|
-
}
|
|
3478
|
-
function structuredPurposeTokens(value) {
|
|
3479
|
-
return value
|
|
3480
|
-
.replace(/([a-z0-9])([A-Z])/g, '$1-$2')
|
|
3481
|
-
.toLowerCase()
|
|
3482
|
-
.split(/[^a-z0-9]+/)
|
|
3483
|
-
.filter(Boolean);
|
|
3484
|
-
}
|
|
3485
|
-
const NON_PRODUCTION_MAVEN_SCOPE_TOKENS = new Set([
|
|
3486
|
-
'benchmark', 'benchmarks', 'checkstyle', 'coverage', 'dev', 'docs', 'integration',
|
|
3487
|
-
'documenter', 'failsafe', 'format', 'jacoco', 'javaformat', 'jmh', 'lint', 'qa', 'site', 'spec',
|
|
3488
|
-
'surefire', 'test', 'tests', 'verification', 'versions',
|
|
3489
|
-
]);
|
|
3490
|
-
function isNonProductionMavenUnit(...values) {
|
|
3491
|
-
return values.some((value) => value !== undefined && structuredPurposeTokens(value)
|
|
3492
|
-
.some((token) => NON_PRODUCTION_MAVEN_SCOPE_TOKENS.has(token)));
|
|
3493
|
-
}
|
|
3494
|
-
function scanMavenManifest(path, lines, records) {
|
|
3495
|
-
if (basename(path).toLowerCase() !== 'pom.xml')
|
|
3496
|
-
return;
|
|
3497
|
-
const source = lines.join('\n');
|
|
3498
|
-
const profiles = xmlBlocks(source, 'profile').map((block) => ({
|
|
3499
|
-
...block,
|
|
3500
|
-
id: xmlValue(block.text, 'id') ?? `profile@${block.line}`,
|
|
3501
|
-
}));
|
|
3502
|
-
const profileAt = (offset) => profiles.find((profile) => offset >= profile.start && offset < profile.end)?.id;
|
|
3503
|
-
const lineAnchor = (line) => lines[line - 1] ?? '<project>';
|
|
3504
|
-
const beforeBuildSections = source
|
|
3505
|
-
.replace(/<parent(?:\s[^>]*)?>[\s\S]*?<\/parent>/, '')
|
|
3506
|
-
.split(/<(?:dependencies|dependencyManagement|build|profiles|modules)(?:\s|>)/, 1)[0];
|
|
3507
|
-
const groupId = xmlValue(beforeBuildSections, 'groupId') ?? 'inherited-group';
|
|
3508
|
-
const artifactId = xmlValue(beforeBuildSections, 'artifactId')
|
|
3509
|
-
?? (normalizePath(path).replace(/\/pom\.xml$/, '') || 'root-project');
|
|
3510
|
-
const version = xmlValue(beforeBuildSections, 'version') ?? 'inherited-version';
|
|
3511
|
-
const packaging = xmlValue(beforeBuildSections, 'packaging') ?? 'jar';
|
|
3512
|
-
const moduleIdentity = `${path}#maven-project:${groupId}:${artifactId}:${version}`;
|
|
3513
|
-
if (packaging !== 'pom')
|
|
3514
|
-
addRecord(records, {
|
|
3515
|
-
kind: 'maven-product-distribution-state',
|
|
3516
|
-
projections: ['stateful_resources'],
|
|
3517
|
-
presumptiveInventory: true,
|
|
3518
|
-
path,
|
|
3519
|
-
line: 1,
|
|
3520
|
-
anchor: lines.find((line) => line.includes('<artifactId>')) ?? lines[0] ?? '<project>',
|
|
3521
|
-
owner: 'maven-project',
|
|
3522
|
-
identity: `distribution:${groupId}:${artifactId}:${version}:${packaging}`,
|
|
3523
|
-
tokens: [groupId, artifactId, version, packaging],
|
|
3524
|
-
addressToken: `${groupId}:${artifactId}`,
|
|
3525
|
-
packageName: `${groupId}:${artifactId}`,
|
|
3526
|
-
packageVersion: version,
|
|
3527
|
-
packageArtifactKind: packaging,
|
|
3528
|
-
materializationRootIdentityKey: moduleIdentity,
|
|
3529
|
-
reproductionOperationToken: 'mvn package',
|
|
3530
|
-
});
|
|
3531
|
-
profiles.filter((profile) => !isNonProductionMavenUnit(profile.id)).forEach((profile) => addRecord(records, {
|
|
3532
|
-
kind: 'maven-installation-profile-state',
|
|
3533
|
-
projections: ['stateful_resources'],
|
|
3534
|
-
presumptiveInventory: true,
|
|
3535
|
-
path,
|
|
3536
|
-
line: profile.line,
|
|
3537
|
-
anchor: lineAnchor(profile.line),
|
|
3538
|
-
owner: 'maven-profiles',
|
|
3539
|
-
identity: `profile:${moduleIdentity}:${profile.id}`,
|
|
3540
|
-
tokens: [moduleIdentity, profile.id],
|
|
3541
|
-
selectorKey: profile.id,
|
|
3542
|
-
addressToken: `${groupId}:${artifactId}:${profile.id}`,
|
|
3543
|
-
materializationRootIdentityKey: `${moduleIdentity}:profile:${profile.id}`,
|
|
3544
|
-
reproductionOperationToken: `mvn -P${profile.id} package`,
|
|
3545
|
-
}));
|
|
3546
|
-
const pluginBlocks = xmlBlocks(source, 'plugin');
|
|
3547
|
-
const enclosingPlugin = (block) => pluginBlocks.find((plugin) => block.start > plugin.start && block.end < plugin.end);
|
|
3548
|
-
xmlBlocks(source, 'dependency').forEach((dependency) => {
|
|
3549
|
-
const dependencyGroup = xmlValue(dependency.text, 'groupId') ?? 'inherited-group';
|
|
3550
|
-
const dependencyArtifact = xmlValue(dependency.text, 'artifactId');
|
|
3551
|
-
if (!dependencyArtifact)
|
|
3552
|
-
return;
|
|
3553
|
-
const dependencyVersion = xmlValue(dependency.text, 'version') ?? 'managed-version';
|
|
3554
|
-
const scope = xmlValue(dependency.text, 'scope') ?? 'compile';
|
|
3555
|
-
if (scope === 'test' || isNonProductionMavenUnit(scope))
|
|
3556
|
-
return;
|
|
3557
|
-
const profile = profileAt(dependency.start) ?? 'base';
|
|
3558
|
-
const plugin = enclosingPlugin(dependency);
|
|
3559
|
-
const type = xmlValue(dependency.text, 'type') ?? 'jar';
|
|
3560
|
-
const classifier = xmlValue(dependency.text, 'classifier') ?? 'default';
|
|
3561
|
-
const optional = xmlValue(dependency.text, 'optional') ?? 'false';
|
|
3562
|
-
const coordinate = `${dependencyGroup}:${dependencyArtifact}:${type}:${classifier}:${dependencyVersion}`;
|
|
3563
|
-
addRecord(records, {
|
|
3564
|
-
kind: plugin ? 'maven-plugin-dependency-requirement-state' : 'maven-dependency-requirement-state',
|
|
3565
|
-
projections: ['stateful_resources'],
|
|
3566
|
-
presumptiveInventory: true,
|
|
3567
|
-
path,
|
|
3568
|
-
line: dependency.line,
|
|
3569
|
-
anchor: lineAnchor(dependency.line),
|
|
3570
|
-
owner: `maven-profile:${profile}`,
|
|
3571
|
-
identity: `requirement:${moduleIdentity}:${profile}:${scope}:${coordinate}:optional=${optional}`,
|
|
3572
|
-
tokens: [moduleIdentity, profile, scope, coordinate, optional],
|
|
3573
|
-
addressToken: coordinate,
|
|
3574
|
-
packageName: `${dependencyGroup}:${dependencyArtifact}`,
|
|
3575
|
-
packageVersion: dependencyVersion,
|
|
3576
|
-
packageArtifactKind: type,
|
|
3577
|
-
materializationRootIdentityKey: `${moduleIdentity}:profile:${profile}`,
|
|
3578
|
-
reproductionOperationToken: profile === 'base' ? 'mvn package' : `mvn -P${profile} package`,
|
|
3579
|
-
});
|
|
3580
|
-
});
|
|
3581
|
-
pluginBlocks.forEach((plugin) => {
|
|
3582
|
-
const pluginGroup = xmlValue(plugin.text, 'groupId') ?? 'org.apache.maven.plugins';
|
|
3583
|
-
const pluginArtifact = xmlValue(plugin.text, 'artifactId');
|
|
3584
|
-
if (!pluginArtifact)
|
|
3585
|
-
return;
|
|
3586
|
-
const pluginVersion = xmlValue(plugin.text, 'version') ?? 'managed-version';
|
|
3587
|
-
const profile = profileAt(plugin.start) ?? 'base';
|
|
3588
|
-
const pluginCoordinate = `${pluginGroup}:${pluginArtifact}:${pluginVersion}`;
|
|
3589
|
-
if (!isNonProductionMavenUnit(pluginArtifact))
|
|
3590
|
-
addRecord(records, {
|
|
3591
|
-
kind: 'maven-build-plugin-requirement-state',
|
|
3592
|
-
projections: ['stateful_resources'],
|
|
3593
|
-
presumptiveInventory: true,
|
|
3594
|
-
path,
|
|
3595
|
-
line: plugin.line,
|
|
3596
|
-
anchor: lineAnchor(plugin.line),
|
|
3597
|
-
owner: `maven-profile:${profile}`,
|
|
3598
|
-
identity: `plugin:${moduleIdentity}:${profile}:${pluginCoordinate}`,
|
|
3599
|
-
tokens: [moduleIdentity, profile, pluginCoordinate],
|
|
3600
|
-
addressToken: pluginCoordinate,
|
|
3601
|
-
packageName: `${pluginGroup}:${pluginArtifact}`,
|
|
3602
|
-
packageVersion: pluginVersion,
|
|
3603
|
-
packageArtifactKind: 'maven-plugin',
|
|
3604
|
-
materializationRootIdentityKey: `${moduleIdentity}:profile:${profile}`,
|
|
3605
|
-
reproductionOperationToken: profile === 'base' ? 'mvn package' : `mvn -P${profile} package`,
|
|
3606
|
-
});
|
|
3607
|
-
const executions = xmlBlocks(plugin.text, 'execution');
|
|
3608
|
-
const admittedExecutions = executions.map((execution) => ({
|
|
3609
|
-
...execution,
|
|
3610
|
-
absoluteLine: plugin.line + execution.line - 1,
|
|
3611
|
-
id: xmlValue(execution.text, 'id') ?? `execution@${plugin.line + execution.line - 1}`,
|
|
3612
|
-
phase: xmlValue(execution.text, 'phase'),
|
|
3613
|
-
goals: xmlBlocks(execution.text, 'goal').map((goal) => goal.text.trim()).filter(Boolean),
|
|
3614
|
-
})).filter((execution) => !isNonProductionMavenUnit(pluginArtifact, execution.id, execution.phase, ...execution.goals));
|
|
3615
|
-
if (admittedExecutions.length === 0)
|
|
3616
|
-
return;
|
|
3617
|
-
const familyIdentity = `plugin-executions:${moduleIdentity}:${profile}:${pluginCoordinate}`;
|
|
3618
|
-
addRecord(records, {
|
|
3619
|
-
kind: 'maven-plugin-execution-family',
|
|
3620
|
-
projections: ['registry_families'],
|
|
3621
|
-
presumptiveInventory: true,
|
|
3622
|
-
path,
|
|
3623
|
-
line: plugin.line,
|
|
3624
|
-
anchor: lineAnchor(plugin.line),
|
|
3625
|
-
owner: `maven-plugin:${pluginCoordinate}`,
|
|
3626
|
-
identity: familyIdentity,
|
|
3627
|
-
tokens: [pluginCoordinate, profile],
|
|
3628
|
-
implementationKey: pluginCoordinate,
|
|
3629
|
-
});
|
|
3630
|
-
admittedExecutions.forEach((execution) => addRecord(records, {
|
|
3631
|
-
kind: 'maven-plugin-execution-entrypoint-state',
|
|
3632
|
-
projections: ['stateful_resources', 'registry_entries'],
|
|
3633
|
-
presumptiveInventory: true,
|
|
3634
|
-
path,
|
|
3635
|
-
line: execution.absoluteLine,
|
|
3636
|
-
anchor: lineAnchor(execution.absoluteLine),
|
|
3637
|
-
owner: `maven-plugin:${pluginCoordinate}`,
|
|
3638
|
-
identity: `${familyIdentity}:${execution.id}`,
|
|
3639
|
-
tokens: [pluginCoordinate, execution.id, execution.phase ?? '', ...execution.goals],
|
|
3640
|
-
selectorKey: execution.id,
|
|
3641
|
-
implementationKey: `${pluginCoordinate}:${execution.goals.join('+') || 'default-goal'}`,
|
|
3642
|
-
addressToken: execution.id,
|
|
3643
|
-
materializationRootIdentityKey: `${moduleIdentity}:profile:${profile}`,
|
|
3644
|
-
reproductionOperationToken: profile === 'base'
|
|
3645
|
-
? `mvn ${pluginGroup}:${pluginArtifact}:${execution.goals[0] ?? execution.id}`
|
|
3646
|
-
: `mvn -P${profile} ${pluginGroup}:${pluginArtifact}:${execution.goals[0] ?? execution.id}`,
|
|
3647
|
-
}));
|
|
3648
|
-
});
|
|
3649
|
-
}
|
|
3650
|
-
function addDeclaredProductionPackage(path, line, anchor, rawName, version, profileIdentityKey, records) {
|
|
3651
|
-
const name = normalizePackageName(rawName);
|
|
3652
|
-
if (!name || name === 'python')
|
|
3653
|
-
return;
|
|
3654
|
-
const alreadyDeclared = [...records.values()].some((record) => record.kind === 'declared-production-package-state' &&
|
|
3655
|
-
record.package_name === name &&
|
|
3656
|
-
record.package_version === version &&
|
|
3657
|
-
record.materialization_root_identity_key === profileIdentityKey);
|
|
3658
|
-
if (alreadyDeclared)
|
|
3659
|
-
return;
|
|
3660
|
-
addRecord(records, {
|
|
3661
|
-
kind: 'declared-production-package-state', projections: ['stateful_resources'], presumptiveInventory: true,
|
|
3662
|
-
path, line, anchor, owner: `production-package-profile:${profileIdentityKey}`,
|
|
3663
|
-
identity: `package:${profileIdentityKey}:${name}@${version}`, tokens: [profileIdentityKey, name, version],
|
|
3664
|
-
addressToken: name, packageName: name, packageVersion: version,
|
|
3665
|
-
materializationRootIdentityKey: profileIdentityKey,
|
|
3666
|
-
});
|
|
3667
|
-
addRecord(records, {
|
|
3668
|
-
kind: 'declared-production-package-install-operation', projections: ['external_dependencies'], presumptiveInventory: true,
|
|
3669
|
-
path, line, anchor, owner: `production-package-profile:${profileIdentityKey}`,
|
|
3670
|
-
identity: `install:${profileIdentityKey}:${name}@${version}`, tokens: [profileIdentityKey, name, version],
|
|
3671
|
-
operationToken: 'install-production-package', packageName: name, packageVersion: version,
|
|
3672
|
-
materializationRootIdentityKey: profileIdentityKey,
|
|
3673
|
-
operationIdentityKey: `${profileIdentityKey}:install:${name}@${version}`,
|
|
3674
|
-
});
|
|
3675
|
-
}
|
|
3676
|
-
function scanProductionManifestPackages(path, lines, records) {
|
|
3677
|
-
const lowerName = basename(path).toLowerCase();
|
|
3678
|
-
if (lowerName === 'pyproject.toml') {
|
|
3679
|
-
let projectSection = false;
|
|
3680
|
-
let projectName = 'python-project';
|
|
3681
|
-
for (const line of lines) {
|
|
3682
|
-
const header = line.trim().match(/^\[([^\]]+)\]$/)?.[1];
|
|
3683
|
-
if (header) {
|
|
3684
|
-
projectSection = header === 'project';
|
|
3685
|
-
continue;
|
|
3686
|
-
}
|
|
3687
|
-
if (!projectSection)
|
|
3688
|
-
continue;
|
|
3689
|
-
const declaredName = line.trim().match(/^name\s*=\s*['"]([^'"]+)['"]/)?.[1];
|
|
3690
|
-
if (declaredName) {
|
|
3691
|
-
projectName = normalizePackageName(declaredName);
|
|
3692
|
-
break;
|
|
3693
|
-
}
|
|
3694
|
-
}
|
|
3695
|
-
let section = '';
|
|
3696
|
-
let inProjectArray = false;
|
|
3697
|
-
let optionalGroup = '';
|
|
3698
|
-
let inOptionalArray = false;
|
|
3699
|
-
lines.forEach((line, index) => {
|
|
3700
|
-
const trimmed = line.trim();
|
|
3701
|
-
const header = trimmed.match(/^\[([^\]]+)\]$/)?.[1];
|
|
3702
|
-
if (header) {
|
|
3703
|
-
section = header;
|
|
3704
|
-
inProjectArray = false;
|
|
3705
|
-
inOptionalArray = false;
|
|
3706
|
-
optionalGroup = '';
|
|
3707
|
-
if (header === 'project')
|
|
3708
|
-
addRecord(records, {
|
|
3709
|
-
kind: 'python-installation-profile-state',
|
|
3710
|
-
projections: ['stateful_resources'],
|
|
3711
|
-
presumptiveInventory: true,
|
|
3712
|
-
path,
|
|
3713
|
-
line: index + 1,
|
|
3714
|
-
anchor: line,
|
|
3715
|
-
owner: 'python-installation-profile',
|
|
3716
|
-
identity: `install-profile:${projectName}`,
|
|
3717
|
-
tokens: [projectName],
|
|
3718
|
-
addressToken: projectName,
|
|
3719
|
-
packageName: projectName,
|
|
3720
|
-
packageVersion: 'base-profile',
|
|
3721
|
-
});
|
|
3722
|
-
return;
|
|
3723
|
-
}
|
|
3724
|
-
const projectArray = section === 'project' && /^dependencies\s*=\s*\[/.test(trimmed);
|
|
3725
|
-
if (projectArray) {
|
|
3726
|
-
inProjectArray = true;
|
|
3727
|
-
}
|
|
3728
|
-
if (section === 'project.optional-dependencies') {
|
|
3729
|
-
const group = optionalGroupName(trimmed);
|
|
3730
|
-
if (group) {
|
|
3731
|
-
optionalGroup = group;
|
|
3732
|
-
inOptionalArray = true;
|
|
3733
|
-
if (!isNonProductionOptionalGroup(group))
|
|
3734
|
-
addRecord(records, {
|
|
3735
|
-
kind: 'python-installation-profile-state',
|
|
3736
|
-
projections: ['stateful_resources'],
|
|
3737
|
-
presumptiveInventory: true,
|
|
3738
|
-
path,
|
|
3739
|
-
line: index + 1,
|
|
3740
|
-
anchor: line,
|
|
3741
|
-
owner: 'python-installation-profile',
|
|
3742
|
-
identity: `install-profile:${projectName}[${group}]`,
|
|
3743
|
-
tokens: [projectName, group],
|
|
3744
|
-
addressToken: `${projectName}[${group}]`,
|
|
3745
|
-
packageName: projectName,
|
|
3746
|
-
packageVersion: `extra:${group}`,
|
|
3747
|
-
});
|
|
3748
|
-
}
|
|
3749
|
-
}
|
|
3750
|
-
const includeProjectLine = projectArray || (section === 'project' && inProjectArray);
|
|
3751
|
-
const includeOptionalLine = section === 'project.optional-dependencies' &&
|
|
3752
|
-
inOptionalArray &&
|
|
3753
|
-
optionalGroup.length > 0 &&
|
|
3754
|
-
!isNonProductionOptionalGroup(optionalGroup);
|
|
3755
|
-
const includePoetryLine = section === 'tool.poetry.dependencies';
|
|
3756
|
-
if (includeProjectLine || includeOptionalLine) {
|
|
3757
|
-
const profileIdentityKey = includeOptionalLine
|
|
3758
|
-
? `${path}#project.optional-dependencies.${optionalGroup}`
|
|
3759
|
-
: `${path}#project`;
|
|
3760
|
-
for (const match of line.matchAll(/['"]([A-Za-z0-9_.-]+)(?:\[[^\]]+\])?([^'"]*)['"]/g)) {
|
|
3761
|
-
if (normalizePackageName(match[1]) === projectName)
|
|
3762
|
-
continue;
|
|
3763
|
-
addDeclaredProductionPackage(path, index + 1, line, match[1], match[2].trim() || 'declared', profileIdentityKey, records);
|
|
3764
|
-
}
|
|
3765
|
-
}
|
|
3766
|
-
else if (includePoetryLine) {
|
|
3767
|
-
const dependency = line.match(/^\s*([A-Za-z0-9_.-]+)\s*=\s*(.+)$/);
|
|
3768
|
-
if (dependency)
|
|
3769
|
-
addDeclaredProductionPackage(path, index + 1, line, dependency[1], dependency[2].trim(), `${path}#tool.poetry.dependencies`, records);
|
|
3770
|
-
}
|
|
3771
|
-
if (includeProjectLine && trimmed.includes(']'))
|
|
3772
|
-
inProjectArray = false;
|
|
3773
|
-
if (includeOptionalLine && trimmed.includes(']'))
|
|
3774
|
-
inOptionalArray = false;
|
|
3775
|
-
});
|
|
3776
|
-
}
|
|
3777
|
-
else if (lowerName === 'package.json') {
|
|
3778
|
-
let parsed;
|
|
3779
|
-
try {
|
|
3780
|
-
parsed = JSON.parse(lines.join('\n'));
|
|
3781
|
-
}
|
|
3782
|
-
catch {
|
|
3783
|
-
return;
|
|
3784
|
-
}
|
|
3785
|
-
const dependencies = { ...(parsed.dependencies ?? {}), ...(parsed.optionalDependencies ?? {}) };
|
|
3786
|
-
Object.entries(dependencies).forEach(([name, version]) => {
|
|
3787
|
-
const lineIndex = lines.findIndex((line) => line.includes(JSON.stringify(name)));
|
|
3788
|
-
const line = lineIndex >= 0 ? lineIndex + 1 : 1;
|
|
3789
|
-
addDeclaredProductionPackage(path, line, lines[line - 1] ?? '{', name, version, `${path}#dependencies`, records);
|
|
3790
|
-
});
|
|
3791
|
-
}
|
|
3792
|
-
else if (lowerName === 'cargo.toml') {
|
|
3793
|
-
let productionSection = false;
|
|
3794
|
-
let productionSectionIdentity = '';
|
|
3795
|
-
lines.forEach((line, index) => {
|
|
3796
|
-
const header = line.trim().match(/^\[([^\]]+)\]$/)?.[1];
|
|
3797
|
-
if (header) {
|
|
3798
|
-
productionSection = /(?:^|\.)(?:dependencies|build-dependencies)$/.test(header) && !/(?:^|\.)dev-dependencies$/.test(header);
|
|
3799
|
-
productionSectionIdentity = productionSection ? `${path}#${header}` : '';
|
|
3800
|
-
return;
|
|
3801
|
-
}
|
|
3802
|
-
if (!productionSection)
|
|
3803
|
-
return;
|
|
3804
|
-
const dependency = line.match(/^\s*([A-Za-z0-9_.-]+)\s*=\s*(.+)$/);
|
|
3805
|
-
if (dependency)
|
|
3806
|
-
addDeclaredProductionPackage(path, index + 1, line, dependency[1], dependency[2].trim(), productionSectionIdentity, records);
|
|
3807
|
-
});
|
|
3808
|
-
}
|
|
3809
|
-
else if (lowerName === 'go.mod') {
|
|
3810
|
-
let requireBlock = false;
|
|
3811
|
-
lines.forEach((line, index) => {
|
|
3812
|
-
const trimmed = line.trim();
|
|
3813
|
-
if (trimmed === 'require (') {
|
|
3814
|
-
requireBlock = true;
|
|
3815
|
-
return;
|
|
3816
|
-
}
|
|
3817
|
-
if (requireBlock && trimmed === ')') {
|
|
3818
|
-
requireBlock = false;
|
|
3819
|
-
return;
|
|
3820
|
-
}
|
|
3821
|
-
const match = (requireBlock ? trimmed : trimmed.replace(/^require\s+/, '')).match(/^([^\s]+)\s+([^\s]+)/);
|
|
3822
|
-
if ((requireBlock || trimmed.startsWith('require ')) && match) {
|
|
3823
|
-
addDeclaredProductionPackage(path, index + 1, line, match[1], match[2], `${path}#require`, records);
|
|
3824
|
-
}
|
|
3825
|
-
});
|
|
3826
|
-
}
|
|
3827
|
-
}
|
|
3828
|
-
function scanManifestEntrypoints(path, lines, records) {
|
|
3829
|
-
const lowerName = basename(path).toLowerCase();
|
|
3830
|
-
if (lowerName === 'pom.xml')
|
|
3831
|
-
scanMavenManifest(path, lines, records);
|
|
3832
|
-
if (lowerName === 'pyproject.toml') {
|
|
3833
|
-
let section = '';
|
|
3834
|
-
lines.forEach((line, index) => {
|
|
3835
|
-
const header = line.trim().match(/^\[(project\.scripts|project\.entry-points\.[^\]]+)\]$/);
|
|
3836
|
-
if (header) {
|
|
3837
|
-
section = header[1];
|
|
3838
|
-
return;
|
|
3839
|
-
}
|
|
3840
|
-
if (/^\[.+\]$/.test(line.trim()))
|
|
3841
|
-
section = '';
|
|
3842
|
-
if (!section)
|
|
3843
|
-
return;
|
|
3844
|
-
const entry = line.match(/^\s*([A-Za-z0-9_.-]+)\s*=\s*['"]([^'"]+)['"]/);
|
|
3845
|
-
if (!entry)
|
|
3846
|
-
return;
|
|
3847
|
-
addRecord(records, {
|
|
3848
|
-
kind: 'installed-entrypoint-state', projections: ['stateful_resources', 'registry_families', 'registry_entries'], presumptiveInventory: true,
|
|
3849
|
-
path, line: index + 1, anchor: line, owner: section, identity: `${entry[1]}:${entry[2]}`,
|
|
3850
|
-
tokens: [entry[1], entry[2]], selectorKey: entry[1], implementationKey: entry[2], addressToken: entry[1],
|
|
3851
|
-
});
|
|
3852
|
-
});
|
|
3853
|
-
scanPythonBuildRequirements(path, lines, records);
|
|
3854
|
-
}
|
|
3855
|
-
if (lowerName === 'package.json') {
|
|
3856
|
-
let parsed;
|
|
3857
|
-
try {
|
|
3858
|
-
parsed = JSON.parse(lines.join('\n'));
|
|
3859
|
-
}
|
|
3860
|
-
catch {
|
|
3861
|
-
return;
|
|
3862
|
-
}
|
|
3863
|
-
const bins = typeof parsed.bin === 'string' ? { [basename(resolve(path), '.json')]: parsed.bin } : parsed.bin ?? {};
|
|
3864
|
-
Object.entries(bins).forEach(([name, implementation]) => {
|
|
3865
|
-
const lineIndex = lines.findIndex((line) => line.includes(JSON.stringify(name)) && line.includes(JSON.stringify(implementation)));
|
|
3866
|
-
const line = lineIndex >= 0 ? lineIndex + 1 : 1;
|
|
3867
|
-
addRecord(records, {
|
|
3868
|
-
kind: 'installed-entrypoint-state', projections: ['stateful_resources', 'registry_families', 'registry_entries'], presumptiveInventory: true,
|
|
3869
|
-
path, line, anchor: lines[line - 1] ?? '{', owner: 'package.bin',
|
|
3870
|
-
identity: `${name}:${implementation}`, tokens: [name, implementation], selectorKey: name,
|
|
3871
|
-
implementationKey: implementation, addressToken: name,
|
|
3872
|
-
});
|
|
3873
|
-
});
|
|
3874
|
-
}
|
|
3875
|
-
scanProductionManifestPackages(path, lines, records);
|
|
3876
|
-
}
|
|
3877
|
-
export function scanUniverseIdentities(repositoryRoot) {
|
|
3878
|
-
const root = resolve(repositoryRoot);
|
|
3879
|
-
const tracked = trackedFiles(root);
|
|
3880
|
-
const requirementProfiles = discoverPythonRequirementProfiles(root, tracked);
|
|
3881
|
-
const pythonLinesByPath = new Map();
|
|
3882
|
-
const pythonCollectionsByPath = new Map();
|
|
3883
|
-
const pythonCollectionsByModule = new Map();
|
|
3884
|
-
for (const path of tracked.filter((candidate) => candidate.endsWith('.py') && shouldScan(candidate))) {
|
|
3885
|
-
let lines = null;
|
|
3886
|
-
try {
|
|
3887
|
-
lines = textLines(root, path);
|
|
3888
|
-
}
|
|
3889
|
-
catch {
|
|
3890
|
-
lines = null;
|
|
3891
|
-
}
|
|
3892
|
-
if (!lines)
|
|
3893
|
-
continue;
|
|
3894
|
-
pythonLinesByPath.set(path, lines);
|
|
3895
|
-
const collections = parsePythonLiteralCollections(path, lines);
|
|
3896
|
-
pythonCollectionsByPath.set(path, collections);
|
|
3897
|
-
pythonCollectionsByModule.set(pythonModuleName(path), collections);
|
|
3898
|
-
}
|
|
3899
|
-
const pythonMatrixAnalyses = new Map();
|
|
3900
|
-
const importedPythonCollectionIdentityKeys = new Set();
|
|
3901
|
-
for (const [path, lines] of pythonLinesByPath) {
|
|
3902
|
-
const bindings = bindPythonLiteralCollections(path, lines, pythonCollectionsByPath.get(path) ?? new Map(), pythonCollectionsByModule);
|
|
3903
|
-
const ownIdentityKeys = new Set([...(pythonCollectionsByPath.get(path) ?? new Map()).values()]
|
|
3904
|
-
.map((collection) => collection.symbolIdentityKey));
|
|
3905
|
-
for (const collection of bindings.values()) {
|
|
3906
|
-
if (!ownIdentityKeys.has(collection.symbolIdentityKey)) {
|
|
3907
|
-
importedPythonCollectionIdentityKeys.add(collection.symbolIdentityKey);
|
|
3908
|
-
}
|
|
3909
|
-
}
|
|
3910
|
-
pythonMatrixAnalyses.set(path, analyzePythonOperationMatrices(path, lines, bindings));
|
|
3911
|
-
}
|
|
3912
|
-
const localPythonModuleRoots = new Set();
|
|
3913
|
-
for (const path of tracked.filter((candidate) => candidate.endsWith('.py'))) {
|
|
3914
|
-
const parts = normalizePath(path).split('/');
|
|
3915
|
-
if (parts[0] === 'src' && parts.length > 2)
|
|
3916
|
-
localPythonModuleRoots.add(parts[1]);
|
|
3917
|
-
else if (parts.length > 1)
|
|
3918
|
-
localPythonModuleRoots.add(parts[0]);
|
|
3919
|
-
else
|
|
3920
|
-
localPythonModuleRoots.add(basename(parts[0], '.py'));
|
|
3921
|
-
}
|
|
3922
|
-
const records = new Map();
|
|
3923
|
-
const omissions = [];
|
|
3924
|
-
const manifestParts = [];
|
|
3925
|
-
let scanned = 0;
|
|
3926
|
-
for (const path of tracked) {
|
|
3927
|
-
const absolute = join(root, path);
|
|
3928
|
-
if (!existsSync(absolute) || !statSync(absolute).isFile())
|
|
3929
|
-
continue;
|
|
3930
|
-
if (!shouldScan(path) && !requirementProfiles.has(path))
|
|
3931
|
-
continue;
|
|
3932
|
-
const bytes = readFileSync(absolute);
|
|
3933
|
-
manifestParts.push(`${path}\0${sha256(bytes)}\n`);
|
|
3934
|
-
let lines;
|
|
3935
|
-
try {
|
|
3936
|
-
lines = pythonLinesByPath.get(path) ?? textLines(root, path);
|
|
3937
|
-
}
|
|
3938
|
-
catch {
|
|
3939
|
-
lines = null;
|
|
3940
|
-
}
|
|
3941
|
-
if (!lines) {
|
|
3942
|
-
omissions.push({ path, reason: 'binary-or-over-size-limit' });
|
|
3943
|
-
continue;
|
|
3944
|
-
}
|
|
3945
|
-
scanned += 1;
|
|
3946
|
-
const lowerName = basename(path).toLowerCase();
|
|
3947
|
-
const extension = extname(lowerName);
|
|
3948
|
-
const owners = extension === '.py' || extension === '.pyi'
|
|
3949
|
-
? pythonOwners(path, lines)
|
|
3950
|
-
: lines.map(() => basename(path));
|
|
3951
|
-
const executable = executableLineMask(path, lines);
|
|
3952
|
-
scanEffects(path, lines, owners, executable, records);
|
|
3953
|
-
scanExternalCallBlocks(path, lines, owners, executable, records);
|
|
3954
|
-
scanJavaAndBrowserOperationalIdentities(path, lines, records);
|
|
3955
|
-
scanWorkflowAndRecipeOperations(path, lines, records);
|
|
3956
|
-
if (extension === '.py') {
|
|
3957
|
-
scanPythonExecutableEntrypoint(path, lines, records);
|
|
3958
|
-
scanPythonEmbeddedConfiguration(path, lines, pythonCollectionsByPath.get(path) ?? new Map(), importedPythonCollectionIdentityKeys, records);
|
|
3959
|
-
scanPythonSelectors(path, lines, owners, executable, records);
|
|
3960
|
-
scanPythonProtocolCallbacks(path, lines, owners, executable, records);
|
|
3961
|
-
scanPythonOptionalImports(path, lines, owners, executable, records);
|
|
3962
|
-
scanPythonImportedSdkOperations(path, lines, owners, executable, localPythonModuleRoots, records);
|
|
3963
|
-
}
|
|
3964
|
-
if (['.js', '.jsx', '.mjs', '.cjs', '.ts', '.tsx', '.mts', '.cts', '.java', '.kt', '.kts', '.cs', '.go', '.rs'].includes(extension)) {
|
|
3965
|
-
scanGenericSelectors(path, lines, executable, records);
|
|
3966
|
-
}
|
|
3967
|
-
scanManifestEntrypoints(path, lines, records);
|
|
3968
|
-
const requirementProfile = requirementProfiles.get(path);
|
|
3969
|
-
if (requirementProfile)
|
|
3970
|
-
scanPythonRequirementProfile(requirementProfile, lines, records);
|
|
3971
|
-
if (lowerName === 'uv.lock')
|
|
3972
|
-
scanUvLock(path, lines, records);
|
|
3973
|
-
if (lowerName === 'package-lock.json' || lowerName === 'npm-shrinkwrap.json') {
|
|
3974
|
-
scanPackageLock(path, lines.join('\n'), records);
|
|
3975
|
-
}
|
|
3976
|
-
}
|
|
3977
|
-
for (const analysis of pythonMatrixAnalyses.values()) {
|
|
3978
|
-
emitPythonExactOperationMatrices(analysis, records);
|
|
3979
|
-
}
|
|
3980
|
-
scanVueSelectorCensus(root, tracked, records);
|
|
3981
|
-
scanJavaSelectorCensus(root, tracked, records);
|
|
3982
|
-
scanJavaExactOperationMatrices(root, tracked, records);
|
|
3983
|
-
scanPascalSelectorIdentities(root, tracked).forEach((record) => addRecord(records, record));
|
|
3984
|
-
scanPythonManifestSelectorIdentities(root, tracked).forEach((record) => addRecord(records, record));
|
|
3985
|
-
scanSpecializedOperationalIdentities(root, tracked).forEach((record) => addRecord(records, record));
|
|
3986
|
-
const sortedRecords = [...records.values()].sort((left, right) => left.id.localeCompare(right.id));
|
|
3987
|
-
const byKind = {};
|
|
3988
|
-
const byProjection = {};
|
|
3989
|
-
const byProofStatus = {
|
|
3990
|
-
'proof-complete': 0,
|
|
3991
|
-
'discovery-lead': 0,
|
|
3992
|
-
'non-inventory': 0,
|
|
3993
|
-
};
|
|
3994
|
-
sortedRecords.forEach((record) => {
|
|
3995
|
-
byKind[record.kind] = (byKind[record.kind] ?? 0) + 1;
|
|
3996
|
-
byProofStatus[record.proof_status] += 1;
|
|
3997
|
-
record.projections.forEach((projection) => {
|
|
3998
|
-
byProjection[projection] = (byProjection[projection] ?? 0) + 1;
|
|
3999
|
-
});
|
|
4000
|
-
});
|
|
4001
|
-
// Hash incrementally so a large (but bounded per-record) census does not need
|
|
4002
|
-
// a second repository-sized string allocation just to compute its receipt.
|
|
4003
|
-
// Preserve the prior canonical encoding exactly: records are separated by a
|
|
4004
|
-
// single newline, with no trailing newline.
|
|
4005
|
-
const recordDigest = createHash('sha256');
|
|
4006
|
-
sortedRecords.forEach((record, index) => {
|
|
4007
|
-
if (index > 0)
|
|
4008
|
-
recordDigest.update('\n');
|
|
4009
|
-
recordDigest.update(JSON.stringify(record));
|
|
4010
|
-
});
|
|
4011
|
-
const projectionRecordIds = Object.fromEntries(['runtime_roles', 'stateful_resources', 'external_dependencies', 'registry_families', 'registry_entries', 'registry_edges']
|
|
4012
|
-
.map((projection) => [
|
|
4013
|
-
projection,
|
|
4014
|
-
sortedRecords
|
|
4015
|
-
.filter((record) => record.presumptive_inventory && record.projections.includes(projection))
|
|
4016
|
-
.map((record) => record.id),
|
|
4017
|
-
]));
|
|
4018
|
-
// Run the data-store pass before building the result so any inputs it cannot
|
|
4019
|
-
// read are appended to `omissions` before `files_omitted` counts them.
|
|
4020
|
-
const dataStores = collectDataStores(root, tracked, omissions);
|
|
4021
|
-
return {
|
|
4022
|
-
schema_version: IDENTITY_CENSUS_SCHEMA_VERSION,
|
|
4023
|
-
contract_revision: IDENTITY_CENSUS_CONTRACT_REVISION,
|
|
4024
|
-
repository_root: root,
|
|
4025
|
-
source_commit: gitObjectId(root, 'HEAD^{commit}'),
|
|
4026
|
-
source_tree: gitObjectId(root, 'HEAD^{tree}'),
|
|
4027
|
-
source_manifest: {
|
|
4028
|
-
digest: `sha256:${sha256(manifestParts.sort().join(''))}`,
|
|
4029
|
-
tracked_files_total: tracked.length,
|
|
4030
|
-
production_files_scanned: scanned,
|
|
4031
|
-
files_omitted: omissions.length,
|
|
4032
|
-
},
|
|
4033
|
-
record_digest: `sha256:${recordDigest.digest('hex')}`,
|
|
4034
|
-
counts: {
|
|
4035
|
-
records: sortedRecords.length,
|
|
4036
|
-
by_kind: Object.fromEntries(Object.entries(byKind).sort(([left], [right]) => left.localeCompare(right))),
|
|
4037
|
-
by_projection: Object.fromEntries(Object.entries(byProjection).sort(([left], [right]) => left.localeCompare(right))),
|
|
4038
|
-
by_proof_status: byProofStatus,
|
|
4039
|
-
presumptive_inventory: sortedRecords.filter((record) => record.presumptive_inventory).length,
|
|
4040
|
-
},
|
|
4041
|
-
projection_record_ids: projectionRecordIds,
|
|
4042
|
-
records: sortedRecords,
|
|
4043
|
-
omissions,
|
|
4044
|
-
data_stores: dataStores,
|
|
4045
|
-
};
|
|
4046
|
-
}
|
|
4047
|
-
export function cloudVerifierIdentityCensusCommand(options) {
|
|
4048
|
-
const root = options.root ? resolve(options.root) : (findGitRoot() ?? process.cwd());
|
|
4049
|
-
const result = scanUniverseIdentities(root);
|
|
4050
|
-
if (options.json) {
|
|
4051
|
-
process.stdout.write(`${JSON.stringify(result, null, 2)}\n`);
|
|
4052
|
-
return;
|
|
4053
|
-
}
|
|
4054
|
-
process.stdout.write(`Cloud Verifier identity census: ${result.counts.records} records from `
|
|
4055
|
-
+ `${result.source_manifest.production_files_scanned}/${result.source_manifest.tracked_files_total} files\n`);
|
|
4056
|
-
process.stdout.write(`Record digest: ${result.record_digest}\n`);
|
|
4057
|
-
Object.entries(result.counts.by_projection).forEach(([projection, count]) => {
|
|
4058
|
-
process.stdout.write(` ${projection}: ${count}\n`);
|
|
4059
|
-
});
|
|
4060
|
-
}
|