@dailephd/my-dev-kit-lab 0.3.0 → 0.3.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +19 -6
- package/dist/scripts/audits/runAudit.js +9 -2
- package/dist/scripts/security/validate.js +8 -29
- package/dist/src/audits/codeRot/detectors/deadCodeCandidateDetector.js +138 -11
- package/dist/src/audits/codeRot/detectors/duplicateImplementationDetector.js +93 -1
- package/dist/src/audits/codeRot/detectors/testRotDetector.js +176 -0
- package/dist/src/audits/codeRot/utils/filePatternUtils.js +50 -0
- package/dist/src/audits/codeRot/utils/sourceFactsLookup.js +53 -0
- package/dist/src/audits/core/auditConfig.js +2 -2
- package/dist/src/audits/core/auditRunner.js +23 -1
- package/dist/src/audits/core/auditTypes.js +14 -7
- package/dist/src/audits/core/collectSourceFacts.js +142 -0
- package/dist/src/audits/core/languageAnalyzerRegistry.js +36 -0
- package/dist/src/audits/core/projectInventory.js +185 -3
- package/dist/src/audits/core/pythonAnalyzer.js +328 -0
- package/dist/src/audits/core/pythonProjectMetadata.js +80 -0
- package/dist/src/audits/core/sourceFacts.js +17 -0
- package/dist/src/audits/core/typescriptJavaScriptAnalyzer.js +299 -0
- package/dist/src/audits/report/auditReportModel.js +15 -0
- package/dist/src/audits/report/renderAuditJsonReport.js +3 -0
- package/dist/src/audits/report/renderAuditTextReport.js +59 -3
- package/dist/src/audits/security/mapSecurityFindingToAuditIssue.js +70 -0
- package/dist/src/audits/security/securityAuditAdapter.js +67 -0
- package/dist/src/audits/security/securityAuditTypes.js +17 -0
- package/dist/src/securityValidation/report/buildSecurityReport.js +31 -0
- package/dist/src/securityValidation/validate/resolveTarget.js +9 -0
- package/package.json +97 -96
|
@@ -27,6 +27,112 @@ export const INVENTORY_FILE_CATEGORIES = [
|
|
|
27
27
|
"report",
|
|
28
28
|
"unknown",
|
|
29
29
|
];
|
|
30
|
+
// ---------------------------------------------------------------------------
|
|
31
|
+
// v0.3.1 Batch 1 — normalized language and file-role metadata.
|
|
32
|
+
//
|
|
33
|
+
// Purely additive to the v0.3.0 category model above: `category` keeps its
|
|
34
|
+
// existing meaning and existing consumers (detectors, report model) are
|
|
35
|
+
// untouched. `language` and `role` are deterministic, extension/path-based
|
|
36
|
+
// classifications layered on top so later v0.3.1 batches (language analyzer
|
|
37
|
+
// registry, per-language code-rot detectors) can select files by language
|
|
38
|
+
// without re-deriving it from extensions themselves. Language detection is
|
|
39
|
+
// classification-only here -- it does not imply parsing/analysis support for
|
|
40
|
+
// python/java/kotlin exists yet.
|
|
41
|
+
// ---------------------------------------------------------------------------
|
|
42
|
+
export const NORMALIZED_LANGUAGES = [
|
|
43
|
+
"typescript",
|
|
44
|
+
"javascript",
|
|
45
|
+
"python",
|
|
46
|
+
"java",
|
|
47
|
+
"kotlin",
|
|
48
|
+
"json",
|
|
49
|
+
"markdown",
|
|
50
|
+
"yaml",
|
|
51
|
+
"xml",
|
|
52
|
+
"toml",
|
|
53
|
+
"unknown",
|
|
54
|
+
];
|
|
55
|
+
const EXTENSION_TO_LANGUAGE = {
|
|
56
|
+
".ts": "typescript",
|
|
57
|
+
".tsx": "typescript",
|
|
58
|
+
".mts": "typescript",
|
|
59
|
+
".cts": "typescript",
|
|
60
|
+
".js": "javascript",
|
|
61
|
+
".jsx": "javascript",
|
|
62
|
+
".mjs": "javascript",
|
|
63
|
+
".cjs": "javascript",
|
|
64
|
+
".py": "python",
|
|
65
|
+
".java": "java",
|
|
66
|
+
".kt": "kotlin",
|
|
67
|
+
".kts": "kotlin",
|
|
68
|
+
".json": "json",
|
|
69
|
+
".md": "markdown",
|
|
70
|
+
".markdown": "markdown",
|
|
71
|
+
".yml": "yaml",
|
|
72
|
+
".yaml": "yaml",
|
|
73
|
+
".xml": "xml",
|
|
74
|
+
".toml": "toml",
|
|
75
|
+
};
|
|
76
|
+
function classifyLanguage(extension) {
|
|
77
|
+
return EXTENSION_TO_LANGUAGE[extension] ?? "unknown";
|
|
78
|
+
}
|
|
79
|
+
export const FILE_ROLES = [
|
|
80
|
+
"source",
|
|
81
|
+
"test",
|
|
82
|
+
"docs",
|
|
83
|
+
"config",
|
|
84
|
+
"package",
|
|
85
|
+
"generated",
|
|
86
|
+
"build-output",
|
|
87
|
+
"vendor",
|
|
88
|
+
"report-output",
|
|
89
|
+
"unknown",
|
|
90
|
+
];
|
|
91
|
+
// Path-segment markers used only when a file wasn't already excluded from
|
|
92
|
+
// traversal entirely (EXCLUDED_DIR_NAMES below) -- e.g. paths reachable from
|
|
93
|
+
// nested benchmark/fixture projects, or directories not yet in the hard
|
|
94
|
+
// exclusion set. Deliberately conservative: matched as whole path segments,
|
|
95
|
+
// not substrings.
|
|
96
|
+
const VENDOR_PATH_SEGMENTS = new Set(["node_modules", "vendor"]);
|
|
97
|
+
const BUILD_OUTPUT_PATH_SEGMENTS = new Set(["dist", "build", "out", ".next", ".turbo", "target", ".gradle"]);
|
|
98
|
+
const GENERATED_PATH_SEGMENTS = new Set([
|
|
99
|
+
"generated",
|
|
100
|
+
".cache",
|
|
101
|
+
".my-dev-kit",
|
|
102
|
+
".my-dev-kit-v1",
|
|
103
|
+
".my-dev-kit-lab",
|
|
104
|
+
]);
|
|
105
|
+
const REPORT_OUTPUT_PATH_SEGMENTS = new Set(["reports", "lab-output"]);
|
|
106
|
+
function classifyFileRole(relativePath, basename, extension, category) {
|
|
107
|
+
const segments = relativePath.split("/");
|
|
108
|
+
if (category === "report" ||
|
|
109
|
+
isStrayReportFile(basename) ||
|
|
110
|
+
segments.some((s) => REPORT_OUTPUT_PATH_SEGMENTS.has(s))) {
|
|
111
|
+
return "report-output";
|
|
112
|
+
}
|
|
113
|
+
if (segments.some((s) => VENDOR_PATH_SEGMENTS.has(s)))
|
|
114
|
+
return "vendor";
|
|
115
|
+
if (extension === ".tsbuildinfo" || segments.some((s) => BUILD_OUTPUT_PATH_SEGMENTS.has(s)))
|
|
116
|
+
return "build-output";
|
|
117
|
+
if (extension === ".map" || segments.some((s) => GENERATED_PATH_SEGMENTS.has(s)))
|
|
118
|
+
return "generated";
|
|
119
|
+
switch (category) {
|
|
120
|
+
case "tests":
|
|
121
|
+
return "test";
|
|
122
|
+
case "docs":
|
|
123
|
+
return "docs";
|
|
124
|
+
case "package":
|
|
125
|
+
return "package";
|
|
126
|
+
case "config":
|
|
127
|
+
case "ci":
|
|
128
|
+
return "config";
|
|
129
|
+
case "source":
|
|
130
|
+
case "scripts":
|
|
131
|
+
return "source";
|
|
132
|
+
default:
|
|
133
|
+
return "unknown";
|
|
134
|
+
}
|
|
135
|
+
}
|
|
30
136
|
// Directory names excluded from traversal entirely -- never descended into.
|
|
31
137
|
const EXCLUDED_DIR_NAMES = new Set([
|
|
32
138
|
"node_modules",
|
|
@@ -96,6 +202,24 @@ const KNOWN_SOURCE_EXTENSIONS = new Set([
|
|
|
96
202
|
".go",
|
|
97
203
|
".rs",
|
|
98
204
|
".java",
|
|
205
|
+
// v0.3.1 Batch 2 fix: .kt/.kts were classified as a NormalizedLanguage
|
|
206
|
+
// ("kotlin") since Batch 1 but were missing here, so a Kotlin file under
|
|
207
|
+
// src/ got category "unknown" (role "unknown") instead of "source" --
|
|
208
|
+
// silently excluding it from source-facts collection. .go/.rs above are
|
|
209
|
+
// intentionally left as "source" despite having no NormalizedLanguage
|
|
210
|
+
// mapping yet (they normalize to language "unknown"), which is the
|
|
211
|
+
// legitimate "known role, unknown language" case collectSourceFacts.ts's
|
|
212
|
+
// fallback policy handles.
|
|
213
|
+
".kt",
|
|
214
|
+
".kts",
|
|
215
|
+
// v0.3.1 Batch 3 fix: same class of gap as the .kt/.kts fix above -- .mts
|
|
216
|
+
// and .cts were mapped to NormalizedLanguage "typescript" since Batch 1
|
|
217
|
+
// but were missing here, so a .mts/.cts file under src/ got category
|
|
218
|
+
// "unknown" (role "unknown") and would never reach the Batch 3
|
|
219
|
+
// TypeScript/JavaScript analyzer's eligibility filter (role source/test
|
|
220
|
+
// only, see collectSourceFacts.ts).
|
|
221
|
+
".mts",
|
|
222
|
+
".cts",
|
|
99
223
|
]);
|
|
100
224
|
// A file this large or larger is skipped entirely (not even stat'd into the
|
|
101
225
|
// inventory) -- protects against pathological huge-file scan cost.
|
|
@@ -123,6 +247,8 @@ export function scanProjectInventory(targetRoot, options = {}) {
|
|
|
123
247
|
}
|
|
124
248
|
const filesByCategory = countByCategory(files);
|
|
125
249
|
const filesByExtension = countByExtension(files);
|
|
250
|
+
const filesByLanguage = countByLanguage(files);
|
|
251
|
+
const filesByRole = countByRole(files);
|
|
126
252
|
const excludedDirectorySummary = [...excludedDirCounts.entries()]
|
|
127
253
|
.map(([name, occurrences]) => ({ name, occurrences }))
|
|
128
254
|
.sort((a, b) => a.name.localeCompare(b.name));
|
|
@@ -134,6 +260,8 @@ export function scanProjectInventory(targetRoot, options = {}) {
|
|
|
134
260
|
skippedFileCount,
|
|
135
261
|
filesByCategory,
|
|
136
262
|
filesByExtension,
|
|
263
|
+
filesByLanguage,
|
|
264
|
+
filesByRole,
|
|
137
265
|
packageFiles: files.filter((f) => f.category === "package"),
|
|
138
266
|
docsFiles: files.filter((f) => f.category === "docs"),
|
|
139
267
|
sourceFiles: files.filter((f) => f.category === "source"),
|
|
@@ -192,11 +320,19 @@ export function scanProjectInventory(targetRoot, options = {}) {
|
|
|
192
320
|
const relativePath = relativeWithinRoot(resolvedRoot, fullPath);
|
|
193
321
|
const extension = path.extname(entry.name).toLowerCase();
|
|
194
322
|
const category = classifyFile(relativePath, entry.name, extension);
|
|
323
|
+
const language = classifyLanguage(extension);
|
|
324
|
+
const role = classifyFileRole(relativePath, entry.name, extension, category);
|
|
195
325
|
const entryResult = {
|
|
196
326
|
relativePath,
|
|
197
327
|
normalizedPath: relativePath.toLowerCase(),
|
|
198
328
|
extension,
|
|
199
329
|
category,
|
|
330
|
+
language,
|
|
331
|
+
role,
|
|
332
|
+
isGenerated: role === "generated",
|
|
333
|
+
isBuildOutput: role === "build-output",
|
|
334
|
+
isVendor: role === "vendor",
|
|
335
|
+
isReportOutput: role === "report-output",
|
|
200
336
|
sizeBytes: stat.size,
|
|
201
337
|
likelyGenerated: category === "generated" || category === "report",
|
|
202
338
|
likelyConfig: category === "config" || category === "package",
|
|
@@ -220,7 +356,7 @@ function classifyFile(relativePath, basename, extension) {
|
|
|
220
356
|
if (relativePath.startsWith(".github/workflows/") && (extension === ".yml" || extension === ".yaml")) {
|
|
221
357
|
return "ci";
|
|
222
358
|
}
|
|
223
|
-
if (isTestFile(relativePath, basename))
|
|
359
|
+
if (isTestFile(relativePath, basename, extension))
|
|
224
360
|
return "tests";
|
|
225
361
|
// Top-level scripts/ directory -- classified before generic "source" so
|
|
226
362
|
// scripts (which are also .ts/.js) land in their own category.
|
|
@@ -238,11 +374,37 @@ function classifyFile(relativePath, basename, extension) {
|
|
|
238
374
|
return "source";
|
|
239
375
|
return "unknown";
|
|
240
376
|
}
|
|
241
|
-
|
|
377
|
+
// v0.3.2 Batch 1 -- conservative pytest naming conventions. Gated to .py so
|
|
378
|
+
// this can never reclassify a non-Python file. "tests/**/*.py" and
|
|
379
|
+
// "**/tests/**/*.py" are already covered by the generic, extension-agnostic
|
|
380
|
+
// check above; only the singular "test/" directory and the test_*.py /
|
|
381
|
+
// *_test.py filename conventions are Python-specific additions here.
|
|
382
|
+
function isPythonTestFile(relativePath, basename) {
|
|
383
|
+
if (relativePath.startsWith("test/") || relativePath.includes("/test/"))
|
|
384
|
+
return true;
|
|
385
|
+
return /^test_.*\.py$/.test(basename) || /^.*_test\.py$/.test(basename);
|
|
386
|
+
}
|
|
387
|
+
function isTestFile(relativePath, basename, extension) {
|
|
242
388
|
if (relativePath.startsWith("tests/") || relativePath.includes("/tests/"))
|
|
243
389
|
return true;
|
|
244
|
-
|
|
390
|
+
if (/\.(test|spec)\.[jt]sx?$/.test(basename))
|
|
391
|
+
return true;
|
|
392
|
+
if (extension === ".py" && isPythonTestFile(relativePath, basename))
|
|
393
|
+
return true;
|
|
394
|
+
return false;
|
|
245
395
|
}
|
|
396
|
+
// v0.3.2 Batch 1 -- recognized Python project/config metadata filenames.
|
|
397
|
+
// Matched by basename only (same convention as PACKAGE_FILE_NAMES above),
|
|
398
|
+
// not restricted to root, since these are specific enough not to collide
|
|
399
|
+
// with unrelated fixture/benchmark content. Deliberately narrow: does not
|
|
400
|
+
// match arbitrary *.toml/*.ini/*.txt files, only these exact names.
|
|
401
|
+
const PYTHON_PROJECT_METADATA_FILE_NAMES = new Set([
|
|
402
|
+
"pyproject.toml",
|
|
403
|
+
"requirements.txt",
|
|
404
|
+
"setup.cfg",
|
|
405
|
+
"tox.ini",
|
|
406
|
+
"pytest.ini",
|
|
407
|
+
]);
|
|
246
408
|
function isConfigFile(relativePath, basename, extension) {
|
|
247
409
|
if (/^tsconfig(\..+)?\.json$/.test(basename))
|
|
248
410
|
return true;
|
|
@@ -252,6 +414,12 @@ function isConfigFile(relativePath, basename, extension) {
|
|
|
252
414
|
return true;
|
|
253
415
|
if (basename === ".editorconfig" || basename === ".gitignore" || basename === ".npmignore")
|
|
254
416
|
return true;
|
|
417
|
+
// v0.3.2 Batch 1 -- setup.py has a .py extension and would otherwise fall
|
|
418
|
+
// through to the generic KNOWN_SOURCE_EXTENSIONS "source" classification
|
|
419
|
+
// below; the other names have no source-extension conflict but are listed
|
|
420
|
+
// here for the same "project metadata, not source" reasoning.
|
|
421
|
+
if (basename === "setup.py" || PYTHON_PROJECT_METADATA_FILE_NAMES.has(basename))
|
|
422
|
+
return true;
|
|
255
423
|
// Root-level dotfile configs (e.g. .prettierrc, .babelrc) not already
|
|
256
424
|
// covered above -- deliberately conservative (root-only, not recursive)
|
|
257
425
|
// to avoid misclassifying dotfiles inside fixture/benchmark projects.
|
|
@@ -297,6 +465,20 @@ function countByCategory(files) {
|
|
|
297
465
|
}
|
|
298
466
|
return counts;
|
|
299
467
|
}
|
|
468
|
+
function countByLanguage(files) {
|
|
469
|
+
const counts = Object.fromEntries(NORMALIZED_LANGUAGES.map((l) => [l, 0]));
|
|
470
|
+
for (const file of files) {
|
|
471
|
+
counts[file.language] += 1;
|
|
472
|
+
}
|
|
473
|
+
return counts;
|
|
474
|
+
}
|
|
475
|
+
function countByRole(files) {
|
|
476
|
+
const counts = Object.fromEntries(FILE_ROLES.map((r) => [r, 0]));
|
|
477
|
+
for (const file of files) {
|
|
478
|
+
counts[file.role] += 1;
|
|
479
|
+
}
|
|
480
|
+
return counts;
|
|
481
|
+
}
|
|
300
482
|
function countByExtension(files) {
|
|
301
483
|
const counts = new Map();
|
|
302
484
|
for (const file of files) {
|
|
@@ -0,0 +1,328 @@
|
|
|
1
|
+
// ---------------------------------------------------------------------------
|
|
2
|
+
// v0.3.2 Batch 1 -- conservative, dependency-free Python source analyzer.
|
|
3
|
+
//
|
|
4
|
+
// There is no Python parser available in this Node/TypeScript project, and
|
|
5
|
+
// per this batch's own scope this analyzer must not add a new parser
|
|
6
|
+
// dependency. Facts are therefore extracted with a deliberately narrow,
|
|
7
|
+
// line-oriented regex scan -- not a real tokenizer, not an AST, and not a
|
|
8
|
+
// syntax validator. This is the single-file, best-effort analogue of
|
|
9
|
+
// typescriptJavaScriptAnalyzer.ts's AST walk: same LanguageAnalyzer contract,
|
|
10
|
+
// same SourceFileFacts shape, same size-bounding philosophy, but confidence
|
|
11
|
+
// is necessarily lower since nothing here proves the file is syntactically
|
|
12
|
+
// valid Python.
|
|
13
|
+
//
|
|
14
|
+
// What this DOES extract (see requirements 3 of the batch spec):
|
|
15
|
+
// - import module[, module2][ as alias]
|
|
16
|
+
// - from [.[.]]module import name[ as alias][, ...] (single physical line)
|
|
17
|
+
// - from module import *
|
|
18
|
+
// - module-level `def`/`async def`/`class` declarations
|
|
19
|
+
// - class-body method declarations, via a simple indentation-based scope
|
|
20
|
+
// stack (not scope-accurate for deeply nested blocks, just "is this def
|
|
21
|
+
// more indented than the nearest enclosing class line")
|
|
22
|
+
// - a single-line `__all__ = [...]`/`(...)` string list, used to set
|
|
23
|
+
// `exported` precisely when present
|
|
24
|
+
//
|
|
25
|
+
// What this explicitly does NOT do (see requirements 3's "do not implement"
|
|
26
|
+
// list): type checking, full AST/semantic resolution, virtualenv/dependency
|
|
27
|
+
// resolution, runtime or dynamic import resolution, decorator/metaclass/
|
|
28
|
+
// fixture semantics, or any claim of dead-code/duplicate-implementation
|
|
29
|
+
// proof. `__import__(...)`/`importlib.import_module(...)` calls are recorded
|
|
30
|
+
// only as an informational diagnostic, never as a resolved import fact.
|
|
31
|
+
// ---------------------------------------------------------------------------
|
|
32
|
+
export const PYTHON_ANALYZER_ID = "python-analyzer";
|
|
33
|
+
const SUPPORTED_EXTENSIONS = [".py"];
|
|
34
|
+
// Mirrors typescriptJavaScriptAnalyzer.ts's MAX_ANALYZABLE_BYTES bound and
|
|
35
|
+
// reasoning -- a large file is not worth line-scanning, and stays at
|
|
36
|
+
// file-level-only rather than file-level-only masquerading as parsed.
|
|
37
|
+
const MAX_ANALYZABLE_BYTES = 1_000_000;
|
|
38
|
+
const IMPORT_LINE_PATTERN = /^\s*import\s+(.+)$/;
|
|
39
|
+
const FROM_IMPORT_LINE_PATTERN = /^\s*from\s+(\.{0,2}[\w.]*)\s+import\s+(.+)$/;
|
|
40
|
+
const DEF_LINE_PATTERN = /^(\s*)(?:async\s+def|def)\s+([A-Za-z_]\w*)\s*\(/;
|
|
41
|
+
const CLASS_LINE_PATTERN = /^(\s*)class\s+([A-Za-z_]\w*)\s*[:(]/;
|
|
42
|
+
const ALL_ASSIGNMENT_PATTERN = /^\s*__all__\s*=\s*[[(](.*)$/;
|
|
43
|
+
const DYNAMIC_IMPORT_PATTERN = /\b(?:__import__\s*\(|importlib\.import_module\s*\()/;
|
|
44
|
+
const QUOTED_NAME_PATTERN = /(['"])((?:(?!\1).)*)\1/g;
|
|
45
|
+
export const PYTHON_ANALYZER = {
|
|
46
|
+
id: PYTHON_ANALYZER_ID,
|
|
47
|
+
supportedLanguages: ["python"],
|
|
48
|
+
supportedExtensions: SUPPORTED_EXTENSIONS,
|
|
49
|
+
analyzeFile,
|
|
50
|
+
};
|
|
51
|
+
function analyzeFile(input) {
|
|
52
|
+
const { relativePath, language, role, content, inventoryEntry } = input;
|
|
53
|
+
if (Buffer.byteLength(content, "utf8") > MAX_ANALYZABLE_BYTES) {
|
|
54
|
+
return {
|
|
55
|
+
relativePath,
|
|
56
|
+
language,
|
|
57
|
+
role,
|
|
58
|
+
parseStatus: "file-level-only",
|
|
59
|
+
analyzerId: PYTHON_ANALYZER_ID,
|
|
60
|
+
lineCount: inventoryEntry.lineCount,
|
|
61
|
+
imports: [],
|
|
62
|
+
exports: [],
|
|
63
|
+
declarations: [],
|
|
64
|
+
references: [],
|
|
65
|
+
diagnostics: [
|
|
66
|
+
{
|
|
67
|
+
severity: "info",
|
|
68
|
+
code: "file-too-large-for-analysis",
|
|
69
|
+
message: `File exceeds the ${MAX_ANALYZABLE_BYTES}-byte analyzer bound; only inventory-level facts are available.`,
|
|
70
|
+
path: relativePath,
|
|
71
|
+
analyzerId: PYTHON_ANALYZER_ID,
|
|
72
|
+
},
|
|
73
|
+
],
|
|
74
|
+
};
|
|
75
|
+
}
|
|
76
|
+
const diagnostics = [];
|
|
77
|
+
const { logical, hadUnterminatedTripleQuote } = buildLogicalLines(content);
|
|
78
|
+
const imports = [];
|
|
79
|
+
for (let i = 0; i < logical.length; i++) {
|
|
80
|
+
const line = logical[i];
|
|
81
|
+
const lineNumber = i + 1;
|
|
82
|
+
if (parseFromImportLine(line, lineNumber, imports, diagnostics, relativePath))
|
|
83
|
+
continue;
|
|
84
|
+
parseImportLine(line, lineNumber, imports);
|
|
85
|
+
}
|
|
86
|
+
const exportedSet = scanAllExport(logical);
|
|
87
|
+
const declarations = scanDeclarations(logical, exportedSet);
|
|
88
|
+
scanDynamicImportPatterns(logical, diagnostics, relativePath);
|
|
89
|
+
const exports = exportedSet ? [...exportedSet].map((name) => ({ name, kind: "named" })) : [];
|
|
90
|
+
if (hadUnterminatedTripleQuote) {
|
|
91
|
+
diagnostics.push({
|
|
92
|
+
severity: "error",
|
|
93
|
+
code: "unterminated-triple-quoted-string",
|
|
94
|
+
message: "File appears to contain an unterminated triple-quoted string (unbalanced \"\"\"/''' delimiters); facts from this best-effort scan may be incomplete.",
|
|
95
|
+
path: relativePath,
|
|
96
|
+
analyzerId: PYTHON_ANALYZER_ID,
|
|
97
|
+
});
|
|
98
|
+
}
|
|
99
|
+
const parseStatus = hadUnterminatedTripleQuote ? "parse-error" : "parsed";
|
|
100
|
+
return {
|
|
101
|
+
relativePath,
|
|
102
|
+
language,
|
|
103
|
+
role,
|
|
104
|
+
parseStatus,
|
|
105
|
+
analyzerId: PYTHON_ANALYZER_ID,
|
|
106
|
+
lineCount: inventoryEntry.lineCount,
|
|
107
|
+
imports,
|
|
108
|
+
exports,
|
|
109
|
+
declarations,
|
|
110
|
+
references: [],
|
|
111
|
+
diagnostics,
|
|
112
|
+
};
|
|
113
|
+
}
|
|
114
|
+
// ---------------------------------------------------------------------------
|
|
115
|
+
// Comment/triple-quoted-string stripping (heuristic, not a tokenizer).
|
|
116
|
+
// ---------------------------------------------------------------------------
|
|
117
|
+
// Strips a trailing `# comment`, respecting simple single/double-quoted
|
|
118
|
+
// strings on the same line (so `x = "a # b"` is not truncated at the `#`
|
|
119
|
+
// inside the string). Does not understand triple-quoted strings -- those are
|
|
120
|
+
// handled one level up by buildLogicalLines(), which never passes a
|
|
121
|
+
// mid-triple-quote-block line to this function.
|
|
122
|
+
function stripLineComment(line) {
|
|
123
|
+
let inSingle = false;
|
|
124
|
+
let inDouble = false;
|
|
125
|
+
for (let i = 0; i < line.length; i++) {
|
|
126
|
+
const ch = line[i];
|
|
127
|
+
if (ch === "\\" && (inSingle || inDouble)) {
|
|
128
|
+
i++;
|
|
129
|
+
continue;
|
|
130
|
+
}
|
|
131
|
+
if (ch === "'" && !inDouble)
|
|
132
|
+
inSingle = !inSingle;
|
|
133
|
+
else if (ch === '"' && !inSingle)
|
|
134
|
+
inDouble = !inDouble;
|
|
135
|
+
else if (ch === "#" && !inSingle && !inDouble)
|
|
136
|
+
return line.slice(0, i);
|
|
137
|
+
}
|
|
138
|
+
return line;
|
|
139
|
+
}
|
|
140
|
+
// Produces one "logical" line per physical line: comments stripped, and any
|
|
141
|
+
// content inside a triple-quoted string block (`"""..."""` / `'''...'''`)
|
|
142
|
+
// blanked out so it can never be misread as an import/def/class statement.
|
|
143
|
+
// Deliberately narrow: only tracks a single open triple-quote marker at a
|
|
144
|
+
// time and does not handle more than one triple-quoted segment opening and
|
|
145
|
+
// closing on the same physical line.
|
|
146
|
+
function buildLogicalLines(content) {
|
|
147
|
+
const rawLines = content.split(/\r\n|\r|\n/);
|
|
148
|
+
const logical = [];
|
|
149
|
+
let openQuote = null;
|
|
150
|
+
for (const rawLine of rawLines) {
|
|
151
|
+
if (openQuote) {
|
|
152
|
+
const closeIdx = rawLine.indexOf(openQuote);
|
|
153
|
+
if (closeIdx === -1) {
|
|
154
|
+
logical.push("");
|
|
155
|
+
continue;
|
|
156
|
+
}
|
|
157
|
+
openQuote = null;
|
|
158
|
+
logical.push(stripLineComment(rawLine.slice(closeIdx + 3)));
|
|
159
|
+
continue;
|
|
160
|
+
}
|
|
161
|
+
const stripped = stripLineComment(rawLine);
|
|
162
|
+
const tripleMatch = stripped.match(/"""|'''/);
|
|
163
|
+
if (!tripleMatch) {
|
|
164
|
+
logical.push(stripped);
|
|
165
|
+
continue;
|
|
166
|
+
}
|
|
167
|
+
const marker = tripleMatch[0];
|
|
168
|
+
const firstIdx = stripped.indexOf(marker);
|
|
169
|
+
const secondIdx = stripped.indexOf(marker, firstIdx + marker.length);
|
|
170
|
+
if (secondIdx === -1) {
|
|
171
|
+
logical.push(stripped.slice(0, firstIdx));
|
|
172
|
+
openQuote = marker;
|
|
173
|
+
}
|
|
174
|
+
else {
|
|
175
|
+
logical.push(stripped.slice(0, firstIdx) + stripped.slice(secondIdx + marker.length));
|
|
176
|
+
}
|
|
177
|
+
}
|
|
178
|
+
return { logical, hadUnterminatedTripleQuote: openQuote !== null };
|
|
179
|
+
}
|
|
180
|
+
function leadingWhitespaceCount(line) {
|
|
181
|
+
let count = 0;
|
|
182
|
+
for (const ch of line) {
|
|
183
|
+
if (ch === " " || ch === "\t")
|
|
184
|
+
count += 1;
|
|
185
|
+
else
|
|
186
|
+
break;
|
|
187
|
+
}
|
|
188
|
+
return count;
|
|
189
|
+
}
|
|
190
|
+
// ---------------------------------------------------------------------------
|
|
191
|
+
// Imports.
|
|
192
|
+
// ---------------------------------------------------------------------------
|
|
193
|
+
function parseImportLine(line, lineNumber, imports) {
|
|
194
|
+
const match = line.match(IMPORT_LINE_PATTERN);
|
|
195
|
+
if (!match)
|
|
196
|
+
return;
|
|
197
|
+
for (const rawSegment of match[1].split(",")) {
|
|
198
|
+
const segment = rawSegment.trim();
|
|
199
|
+
if (!segment)
|
|
200
|
+
continue;
|
|
201
|
+
const asMatch = segment.match(/^([\w.]+)\s+as\s+(\w+)$/);
|
|
202
|
+
if (asMatch) {
|
|
203
|
+
imports.push({ source: asMatch[1], kind: "static", importedNames: [asMatch[2]], line: lineNumber });
|
|
204
|
+
}
|
|
205
|
+
else {
|
|
206
|
+
imports.push({ source: segment, kind: "static", line: lineNumber });
|
|
207
|
+
}
|
|
208
|
+
}
|
|
209
|
+
}
|
|
210
|
+
function parseFromImportLine(line, lineNumber, imports, diagnostics, relativePath) {
|
|
211
|
+
const match = line.match(FROM_IMPORT_LINE_PATTERN);
|
|
212
|
+
if (!match)
|
|
213
|
+
return false;
|
|
214
|
+
const modulePath = match[1] || ".";
|
|
215
|
+
let namesPart = match[2].trim();
|
|
216
|
+
if (namesPart.startsWith("(") && namesPart.endsWith(")")) {
|
|
217
|
+
namesPart = namesPart.slice(1, -1);
|
|
218
|
+
}
|
|
219
|
+
else if (namesPart.startsWith("(")) {
|
|
220
|
+
// Multi-line parenthesized from-import list -- this single-line scanner
|
|
221
|
+
// does not follow continuation lines. Recorded as a diagnostic rather
|
|
222
|
+
// than silently dropped or guessed at.
|
|
223
|
+
diagnostics.push({
|
|
224
|
+
severity: "info",
|
|
225
|
+
code: "multiline-from-import-unsupported",
|
|
226
|
+
message: "Multi-line parenthesized from-import list is not analyzed by this conservative single-line scanner.",
|
|
227
|
+
path: relativePath,
|
|
228
|
+
analyzerId: PYTHON_ANALYZER_ID,
|
|
229
|
+
});
|
|
230
|
+
return true;
|
|
231
|
+
}
|
|
232
|
+
if (namesPart === "*") {
|
|
233
|
+
imports.push({ source: modulePath, kind: "static", line: lineNumber });
|
|
234
|
+
return true;
|
|
235
|
+
}
|
|
236
|
+
const importedNames = [];
|
|
237
|
+
for (const rawSegment of namesPart.split(",")) {
|
|
238
|
+
const segment = rawSegment.trim();
|
|
239
|
+
if (!segment)
|
|
240
|
+
continue;
|
|
241
|
+
const asMatch = segment.match(/^(\w+)\s+as\s+(\w+)$/);
|
|
242
|
+
importedNames.push(asMatch ? asMatch[2] : segment);
|
|
243
|
+
}
|
|
244
|
+
imports.push({
|
|
245
|
+
source: modulePath,
|
|
246
|
+
kind: "static",
|
|
247
|
+
line: lineNumber,
|
|
248
|
+
...(importedNames.length > 0 ? { importedNames } : {}),
|
|
249
|
+
});
|
|
250
|
+
return true;
|
|
251
|
+
}
|
|
252
|
+
// ---------------------------------------------------------------------------
|
|
253
|
+
// __all__ (optional, single-line only).
|
|
254
|
+
// ---------------------------------------------------------------------------
|
|
255
|
+
function scanAllExport(logical) {
|
|
256
|
+
for (const line of logical) {
|
|
257
|
+
const match = line.match(ALL_ASSIGNMENT_PATTERN);
|
|
258
|
+
if (!match)
|
|
259
|
+
continue;
|
|
260
|
+
const names = [...match[1].matchAll(QUOTED_NAME_PATTERN)].map((m) => m[2]);
|
|
261
|
+
if (names.length > 0)
|
|
262
|
+
return new Set(names);
|
|
263
|
+
}
|
|
264
|
+
return null;
|
|
265
|
+
}
|
|
266
|
+
function isExported(name, exportedSet) {
|
|
267
|
+
if (exportedSet)
|
|
268
|
+
return exportedSet.has(name);
|
|
269
|
+
// Conservative convention-based heuristic when no __all__ is present: a
|
|
270
|
+
// leading underscore is Python's own "not public" signal. This is a
|
|
271
|
+
// heuristic, not a semantic guarantee -- see module header comment.
|
|
272
|
+
return !name.startsWith("_");
|
|
273
|
+
}
|
|
274
|
+
// ---------------------------------------------------------------------------
|
|
275
|
+
// Declarations (functions, async functions, classes, class methods).
|
|
276
|
+
// ---------------------------------------------------------------------------
|
|
277
|
+
function scanDeclarations(logical, exportedSet) {
|
|
278
|
+
const declarations = [];
|
|
279
|
+
// Indent levels of currently-open `class` blocks, outermost first. A line
|
|
280
|
+
// whose indentation drops to or below the top of this stack has exited
|
|
281
|
+
// that class body.
|
|
282
|
+
const classIndentStack = [];
|
|
283
|
+
for (let i = 0; i < logical.length; i++) {
|
|
284
|
+
const line = logical[i];
|
|
285
|
+
if (line.trim().length === 0)
|
|
286
|
+
continue;
|
|
287
|
+
const lineNumber = i + 1;
|
|
288
|
+
const indent = leadingWhitespaceCount(line);
|
|
289
|
+
while (classIndentStack.length > 0 && indent <= classIndentStack[classIndentStack.length - 1]) {
|
|
290
|
+
classIndentStack.pop();
|
|
291
|
+
}
|
|
292
|
+
const classMatch = line.match(CLASS_LINE_PATTERN);
|
|
293
|
+
if (classMatch) {
|
|
294
|
+
const name = classMatch[2];
|
|
295
|
+
declarations.push({ name, kind: "class", exported: isExported(name, exportedSet), line: lineNumber });
|
|
296
|
+
classIndentStack.push(indent);
|
|
297
|
+
continue;
|
|
298
|
+
}
|
|
299
|
+
const defMatch = line.match(DEF_LINE_PATTERN);
|
|
300
|
+
if (defMatch) {
|
|
301
|
+
const name = defMatch[2];
|
|
302
|
+
const isMethod = classIndentStack.length > 0 && indent > classIndentStack[classIndentStack.length - 1];
|
|
303
|
+
declarations.push({
|
|
304
|
+
name,
|
|
305
|
+
kind: isMethod ? "method" : "function",
|
|
306
|
+
// Methods are not tracked against __all__/underscore-export
|
|
307
|
+
// semantics -- __all__ only ever names module-level symbols in
|
|
308
|
+
// real Python, so claiming a method is "exported" would overclaim.
|
|
309
|
+
exported: isMethod ? false : isExported(name, exportedSet),
|
|
310
|
+
line: lineNumber,
|
|
311
|
+
});
|
|
312
|
+
}
|
|
313
|
+
}
|
|
314
|
+
return declarations;
|
|
315
|
+
}
|
|
316
|
+
function scanDynamicImportPatterns(logical, diagnostics, relativePath) {
|
|
317
|
+
for (const line of logical) {
|
|
318
|
+
if (!DYNAMIC_IMPORT_PATTERN.test(line))
|
|
319
|
+
continue;
|
|
320
|
+
diagnostics.push({
|
|
321
|
+
severity: "info",
|
|
322
|
+
code: "dynamic-import-pattern-unsupported",
|
|
323
|
+
message: "Detected a __import__/importlib.import_module call; dynamic imports are not resolved as import facts.",
|
|
324
|
+
path: relativePath,
|
|
325
|
+
analyzerId: PYTHON_ANALYZER_ID,
|
|
326
|
+
});
|
|
327
|
+
}
|
|
328
|
+
}
|
|
@@ -0,0 +1,80 @@
|
|
|
1
|
+
import fs from "node:fs";
|
|
2
|
+
import path from "node:path";
|
|
3
|
+
const PYPROJECT_PROJECT_NAME_PATTERN = /^\s*name\s*=\s*["']([^"']+)["']\s*$/m;
|
|
4
|
+
const PYPROJECT_PYTEST_SECTION_PATTERN = /^\s*\[tool\.pytest\.ini_options\]\s*$/m;
|
|
5
|
+
const SETUP_CFG_PYTEST_SECTION_PATTERN = /^\s*\[tool:pytest\]\s*$/m;
|
|
6
|
+
const INI_SECTION_HEADER_PATTERN = /^\s*\[([^\]]+)\]\s*$/;
|
|
7
|
+
const INI_NAME_KEY_PATTERN = /^\s*name\s*=\s*(.+?)\s*$/;
|
|
8
|
+
// Extracts the `name = ...` value from a setup.cfg-style [metadata] section
|
|
9
|
+
// via a simple line scan (no INI parser dependency) -- stops at the next
|
|
10
|
+
// "[section]" header or end of file.
|
|
11
|
+
function extractSetupCfgProjectName(content) {
|
|
12
|
+
const lines = content.split(/\r\n|\r|\n/);
|
|
13
|
+
let inMetadataSection = false;
|
|
14
|
+
for (const line of lines) {
|
|
15
|
+
const sectionMatch = line.match(INI_SECTION_HEADER_PATTERN);
|
|
16
|
+
if (sectionMatch) {
|
|
17
|
+
inMetadataSection = sectionMatch[1].trim().toLowerCase() === "metadata";
|
|
18
|
+
continue;
|
|
19
|
+
}
|
|
20
|
+
if (!inMetadataSection)
|
|
21
|
+
continue;
|
|
22
|
+
const nameMatch = line.match(INI_NAME_KEY_PATTERN);
|
|
23
|
+
if (nameMatch)
|
|
24
|
+
return nameMatch[1];
|
|
25
|
+
}
|
|
26
|
+
return null;
|
|
27
|
+
}
|
|
28
|
+
export function collectPythonProjectMetadata(targetRoot, inventory) {
|
|
29
|
+
const warnings = [];
|
|
30
|
+
const knownPaths = new Set(inventory.files.map((f) => f.relativePath));
|
|
31
|
+
const has = (name) => knownPaths.has(name);
|
|
32
|
+
const hasPyprojectToml = has("pyproject.toml");
|
|
33
|
+
const hasRequirementsTxt = has("requirements.txt");
|
|
34
|
+
const hasSetupPy = has("setup.py");
|
|
35
|
+
const hasSetupCfg = has("setup.cfg");
|
|
36
|
+
const hasToxIni = has("tox.ini");
|
|
37
|
+
const hasPytestIni = has("pytest.ini");
|
|
38
|
+
let projectName = null;
|
|
39
|
+
let pytestSectionDetected = false;
|
|
40
|
+
if (hasPyprojectToml) {
|
|
41
|
+
const content = readSafely(targetRoot, "pyproject.toml", warnings);
|
|
42
|
+
if (content !== null) {
|
|
43
|
+
const nameMatch = content.match(PYPROJECT_PROJECT_NAME_PATTERN);
|
|
44
|
+
if (nameMatch)
|
|
45
|
+
projectName = nameMatch[1];
|
|
46
|
+
if (PYPROJECT_PYTEST_SECTION_PATTERN.test(content))
|
|
47
|
+
pytestSectionDetected = true;
|
|
48
|
+
}
|
|
49
|
+
}
|
|
50
|
+
if (hasSetupCfg) {
|
|
51
|
+
const content = readSafely(targetRoot, "setup.cfg", warnings);
|
|
52
|
+
if (content !== null) {
|
|
53
|
+
if (projectName === null) {
|
|
54
|
+
projectName = extractSetupCfgProjectName(content);
|
|
55
|
+
}
|
|
56
|
+
if (SETUP_CFG_PYTEST_SECTION_PATTERN.test(content))
|
|
57
|
+
pytestSectionDetected = true;
|
|
58
|
+
}
|
|
59
|
+
}
|
|
60
|
+
return {
|
|
61
|
+
hasPyprojectToml,
|
|
62
|
+
hasRequirementsTxt,
|
|
63
|
+
hasSetupPy,
|
|
64
|
+
hasSetupCfg,
|
|
65
|
+
hasToxIni,
|
|
66
|
+
hasPytestIni,
|
|
67
|
+
hasPytestConfiguration: hasPytestIni || pytestSectionDetected,
|
|
68
|
+
projectName,
|
|
69
|
+
warnings,
|
|
70
|
+
};
|
|
71
|
+
}
|
|
72
|
+
function readSafely(targetRoot, relativePath, warnings) {
|
|
73
|
+
try {
|
|
74
|
+
return fs.readFileSync(path.join(targetRoot, relativePath), "utf8");
|
|
75
|
+
}
|
|
76
|
+
catch (err) {
|
|
77
|
+
warnings.push(`Could not read ${relativePath}: ${err instanceof Error ? err.message : String(err)}`);
|
|
78
|
+
return null;
|
|
79
|
+
}
|
|
80
|
+
}
|
|
@@ -0,0 +1,17 @@
|
|
|
1
|
+
// ---------------------------------------------------------------------------
|
|
2
|
+
// v0.3.1 Batch 2 -- normalized source facts model.
|
|
3
|
+
//
|
|
4
|
+
// Purely additive substrate: no concrete language analyzer exists yet (that
|
|
5
|
+
// is Batch 3's TypeScript/JavaScript analyzer), so every SourceFileFacts
|
|
6
|
+
// produced in this batch is either a deterministic fallback (see
|
|
7
|
+
// collectSourceFacts.ts) or a parse-error record. The fact arrays
|
|
8
|
+
// (imports/exports/declarations/references) are always empty in this batch
|
|
9
|
+
// -- the shape exists so Batch 3 can populate them without a type change.
|
|
10
|
+
// ---------------------------------------------------------------------------
|
|
11
|
+
export const SOURCE_FACT_PARSE_STATUSES = [
|
|
12
|
+
"parsed",
|
|
13
|
+
"file-level-only",
|
|
14
|
+
"unsupported",
|
|
15
|
+
"parse-error",
|
|
16
|
+
"skipped",
|
|
17
|
+
];
|