gitnexus 1.6.8 → 1.6.9-rc.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/cli/analyze.js +30 -7
- package/dist/core/ingestion/cfg/visitors/python-harvest.d.ts +9 -11
- package/dist/core/ingestion/cfg/visitors/python-harvest.js +85 -52
- package/dist/core/ingestion/scope-resolution/pipeline/run.js +5 -5
- package/dist/core/ingestion/taint/python-model.d.ts +10 -0
- package/dist/core/ingestion/taint/python-model.js +43 -0
- package/dist/core/ingestion/taint/typescript-model.d.ts +13 -5
- package/dist/core/ingestion/taint/typescript-model.js +20 -7
- package/dist/core/ingestion/workers/parse-worker.js +35 -27
- package/dist/core/lbug/conn-lock.d.ts +1 -0
- package/dist/core/lbug/conn-lock.js +61 -0
- package/dist/core/lbug/lbug-adapter.d.ts +21 -0
- package/dist/core/lbug/lbug-adapter.js +330 -196
- package/dist/core/lbug/shutdown-helpers.d.ts +18 -0
- package/dist/core/lbug/shutdown-helpers.js +36 -0
- package/dist/core/lbug/wal-checkpoint-driver.js +16 -0
- package/dist/core/lbug/wal-driver-state.d.ts +4 -0
- package/dist/core/lbug/wal-driver-state.js +23 -0
- package/dist/core/run-analyze.d.ts +9 -0
- package/dist/core/run-analyze.js +30 -5
- package/dist/server/analyze-job.js +6 -0
- package/dist/server/analyze-launch.js +7 -0
- package/dist/server/analyze-worker-core.d.ts +43 -0
- package/dist/server/analyze-worker-core.js +56 -0
- package/dist/server/analyze-worker.js +63 -32
- package/dist/storage/repo-manager.d.ts +14 -0
- package/dist/storage/repo-manager.js +24 -17
- package/package.json +1 -1
package/dist/cli/analyze.js
CHANGED
|
@@ -12,7 +12,8 @@ import os from 'os';
|
|
|
12
12
|
import { spawn } from 'child_process';
|
|
13
13
|
import v8 from 'v8';
|
|
14
14
|
import cliProgress from 'cli-progress';
|
|
15
|
-
import {
|
|
15
|
+
import { isLbugReady } from '../core/lbug/lbug-adapter.js';
|
|
16
|
+
import { boundedCheckpointBeforeExit } from '../core/lbug/shutdown-helpers.js';
|
|
16
17
|
import { isLbugCheckpointIoError, isWalCorruptionError, parseWalCheckpointThreshold, WAL_RECOVERY_SUGGESTION, } from '../core/lbug/lbug-config.js';
|
|
17
18
|
import { getStoragePaths, getGlobalRegistryPath, RegistryNameCollisionError, AnalysisNotFinalizedError, assertAnalysisFinalized, } from '../storage/repo-manager.js';
|
|
18
19
|
import { getGitRoot, hasGitDir, getDefaultBranch } from '../storage/git.js';
|
|
@@ -508,6 +509,17 @@ export const analyzeCommand = async (inputPath, options) => {
|
|
|
508
509
|
finally {
|
|
509
510
|
restoreAnalyzeEnv(envSnap);
|
|
510
511
|
}
|
|
512
|
+
// If analyzeCommandImpl returned via a soft `process.exitCode = 1` error path
|
|
513
|
+
// while LadybugDB native handles are still open, the event loop won't drain and
|
|
514
|
+
// the process would HANG (#2264 review P1). The full analyze paths skip-close the
|
|
515
|
+
// DB — handles are left open and reclaimed by process.exit — so a soft return
|
|
516
|
+
// after a real analyze must force the exit. The success path never reaches here
|
|
517
|
+
// (analyzeCommandImpl calls process.exit(0) itself); early-validation errors and
|
|
518
|
+
// unit tests that mock runFullAnalysis never open the DB, so isLbugReady() is
|
|
519
|
+
// false and the soft return is preserved.
|
|
520
|
+
if (isLbugReady()) {
|
|
521
|
+
process.exit(typeof process.exitCode === 'number' ? process.exitCode : 1);
|
|
522
|
+
}
|
|
511
523
|
};
|
|
512
524
|
const analyzeCommandImpl = async (inputPath, cliOptions) => {
|
|
513
525
|
console.log('\n GitNexus Analyzer\n');
|
|
@@ -867,12 +879,17 @@ const analyzeCommandImpl = async (inputPath, cliOptions) => {
|
|
|
867
879
|
aborted = true;
|
|
868
880
|
bar.stop();
|
|
869
881
|
console.log('\n Interrupted — cleaning up...');
|
|
870
|
-
|
|
871
|
-
|
|
872
|
-
|
|
873
|
-
|
|
874
|
-
|
|
875
|
-
|
|
882
|
+
// Bounded CHECKPOINT-then-exit (#2264 review P3): skip the native close (the
|
|
883
|
+
// LadybugDB destructor can double-free after --pdg writes), but don't hang
|
|
884
|
+
// behind a long --pdg COPY holding the connection lock — bound it so a single
|
|
885
|
+
// Ctrl-C stays responsive; the WAL replays on the next analyze. A second
|
|
886
|
+
// Ctrl-C (`if (aborted) process.exit(1)` above) remains the escape hatch.
|
|
887
|
+
void boundedCheckpointBeforeExit({
|
|
888
|
+
exitCode: 130,
|
|
889
|
+
beforeExit: async () => {
|
|
890
|
+
const { flushLoggerSync } = await import('../core/logger.js');
|
|
891
|
+
flushLoggerSync();
|
|
892
|
+
},
|
|
876
893
|
});
|
|
877
894
|
};
|
|
878
895
|
process.on('SIGINT', sigintHandler);
|
|
@@ -964,6 +981,12 @@ const analyzeCommandImpl = async (inputPath, cliOptions) => {
|
|
|
964
981
|
// Extra fetch-wrapper names from `.gitnexusrc` (#1589/#1852 residual);
|
|
965
982
|
// forwarded to the routes phase consumer scan.
|
|
966
983
|
fetchWrappers: options.fetchWrappers,
|
|
984
|
+
// The CLI always process.exit()s after this returns (success path at the
|
|
985
|
+
// end of analyzeCommandImpl, error/interrupt paths via process.exit too),
|
|
986
|
+
// so the finalize close skips the native conn/db close — it can double-free
|
|
987
|
+
// in LadybugDB's ClientContext destructor after --pdg writes (#2264). The
|
|
988
|
+
// CHECKPOINT keeps the index durable; process exit reclaims the handles.
|
|
989
|
+
skipNativeCloseOnExit: true,
|
|
967
990
|
}, {
|
|
968
991
|
onProgress: (_phase, percent, message) => {
|
|
969
992
|
updateBar(percent, message);
|
|
@@ -109,10 +109,10 @@ export declare class PythonHarvester {
|
|
|
109
109
|
/** >0 while walking a conditionally-evaluated subexpression — defs become may-defs. */
|
|
110
110
|
private conditionalDepth;
|
|
111
111
|
/**
|
|
112
|
-
* `call` node id → binding indices its
|
|
113
|
-
*
|
|
114
|
-
* (see {@link registerResultDefs}) and consumed by {@link visitCall}.
|
|
115
|
-
* the Go harvester's `resultDefTargets`.
|
|
112
|
+
* `call` node id → binding indices its result is assigned to (`x = f()` ⇒
|
|
113
|
+
* `[x]`, `a, b = f()` ⇒ `[a, b]`). Populated just before the value walk reaches
|
|
114
|
+
* the call (see {@link registerResultDefs}) and consumed by {@link visitCall}.
|
|
115
|
+
* Mirrors the Go harvester's `resultDefTargets`.
|
|
116
116
|
*/
|
|
117
117
|
private readonly resultDefTargets;
|
|
118
118
|
constructor(fnNode: SyntaxNode);
|
|
@@ -184,13 +184,9 @@ export declare class PythonHarvester {
|
|
|
184
184
|
* (it reads outer names), the body/filter after the targets are bound.
|
|
185
185
|
*/
|
|
186
186
|
private walkComprehension;
|
|
187
|
-
/**
|
|
188
|
-
|
|
189
|
-
|
|
190
|
-
* (def-position identifiers). Consumed by {@link visitCall} once the value
|
|
191
|
-
* walk reaches the node. Single-target only (the caller restricts to a plain
|
|
192
|
-
* identifier LHS); the blank identifier (`_`) binds nothing and is skipped.
|
|
193
|
-
*/
|
|
187
|
+
/** Binding indices assigned by a target pattern, without mutating statement facts. */
|
|
188
|
+
private targetDefIndices;
|
|
189
|
+
/** Record result defs for a call-valued assignment. */
|
|
194
190
|
private registerResultDefs;
|
|
195
191
|
/** Strip `parenthesized_expression` wrappers around a value (`(f())`). */
|
|
196
192
|
private unwrapValue;
|
|
@@ -201,6 +197,8 @@ export declare class PythonHarvester {
|
|
|
201
197
|
* Python has no `new` — every site is `kind: 'call'`.
|
|
202
198
|
*/
|
|
203
199
|
private visitCall;
|
|
200
|
+
/** Walk arguments, assigning only positional values to positional sink slots. */
|
|
201
|
+
private walkArgs;
|
|
204
202
|
/**
|
|
205
203
|
* `attribute` chain walk shared by value position and callee position. Records
|
|
206
204
|
* the chain-root identifier as a use (identical to the old default descent)
|
|
@@ -15,10 +15,10 @@ export class PythonHarvester {
|
|
|
15
15
|
/** >0 while walking a conditionally-evaluated subexpression — defs become may-defs. */
|
|
16
16
|
conditionalDepth = 0;
|
|
17
17
|
/**
|
|
18
|
-
* `call` node id → binding indices its
|
|
19
|
-
*
|
|
20
|
-
* (see {@link registerResultDefs}) and consumed by {@link visitCall}.
|
|
21
|
-
* the Go harvester's `resultDefTargets`.
|
|
18
|
+
* `call` node id → binding indices its result is assigned to (`x = f()` ⇒
|
|
19
|
+
* `[x]`, `a, b = f()` ⇒ `[a, b]`). Populated just before the value walk reaches
|
|
20
|
+
* the call (see {@link registerResultDefs}) and consumed by {@link visitCall}.
|
|
21
|
+
* Mirrors the Go harvester's `resultDefTargets`.
|
|
22
22
|
*/
|
|
23
23
|
resultDefTargets = new Map();
|
|
24
24
|
constructor(fnNode) {
|
|
@@ -124,8 +124,20 @@ export class PythonHarvester {
|
|
|
124
124
|
*/
|
|
125
125
|
prescan(node) {
|
|
126
126
|
const t = node.type;
|
|
127
|
-
if (NESTED_FUNCTION_TYPES.has(t) && node.id !== this.fnId)
|
|
127
|
+
if (NESTED_FUNCTION_TYPES.has(t) && node.id !== this.fnId) {
|
|
128
|
+
if (t === 'function_definition') {
|
|
129
|
+
const name = node.childForFieldName('name');
|
|
130
|
+
if (name?.type === 'identifier')
|
|
131
|
+
this.declare(name, 'function');
|
|
132
|
+
}
|
|
128
133
|
return;
|
|
134
|
+
}
|
|
135
|
+
if (t === 'class_definition') {
|
|
136
|
+
const name = node.childForFieldName('name');
|
|
137
|
+
if (name?.type === 'identifier')
|
|
138
|
+
this.declare(name, 'class');
|
|
139
|
+
return;
|
|
140
|
+
}
|
|
129
141
|
switch (t) {
|
|
130
142
|
case 'global_statement':
|
|
131
143
|
case 'nonlocal_statement': {
|
|
@@ -469,10 +481,10 @@ export class PythonHarvester {
|
|
|
469
481
|
const left = node.childForFieldName('left');
|
|
470
482
|
const right = node.childForFieldName('right');
|
|
471
483
|
// Register result-defs BEFORE walking the value so the nested call site
|
|
472
|
-
// (reached during the value walk) carries them
|
|
473
|
-
//
|
|
474
|
-
if (left
|
|
475
|
-
this.registerResultDefs(right,
|
|
484
|
+
// (reached during the value walk) carries them. Plain and unpack targets
|
|
485
|
+
// are preserved; attribute/subscript LHS attaches nothing.
|
|
486
|
+
if (left && right)
|
|
487
|
+
this.registerResultDefs(right, this.targetDefIndices(left));
|
|
476
488
|
if (right)
|
|
477
489
|
this.walkValue(right, acc);
|
|
478
490
|
if (left)
|
|
@@ -500,8 +512,9 @@ export class PythonHarvester {
|
|
|
500
512
|
// walrus `(n := v)` — `n` is a def, `v` a use.
|
|
501
513
|
const name = node.childForFieldName('name');
|
|
502
514
|
const value = node.childForFieldName('value');
|
|
503
|
-
if (name?.type === 'identifier' && value)
|
|
504
|
-
this.registerResultDefs(value,
|
|
515
|
+
if (name?.type === 'identifier' && value) {
|
|
516
|
+
this.registerResultDefs(value, this.targetDefIndices(name));
|
|
517
|
+
}
|
|
505
518
|
if (value)
|
|
506
519
|
this.walkValue(value, acc);
|
|
507
520
|
if (name?.type === 'identifier')
|
|
@@ -607,27 +620,34 @@ export class PythonHarvester {
|
|
|
607
620
|
if (body)
|
|
608
621
|
this.walkValue(body, acc);
|
|
609
622
|
}
|
|
610
|
-
|
|
611
|
-
|
|
612
|
-
|
|
613
|
-
|
|
614
|
-
|
|
615
|
-
|
|
616
|
-
|
|
617
|
-
|
|
618
|
-
|
|
619
|
-
|
|
620
|
-
|
|
621
|
-
return;
|
|
622
|
-
const defs = [];
|
|
623
|
-
for (const target of targets) {
|
|
624
|
-
if (target.type !== 'identifier' || target.text === '_')
|
|
625
|
-
continue;
|
|
626
|
-
defs.push(this.resolve(target));
|
|
623
|
+
/** Binding indices assigned by a target pattern, without mutating statement facts. */
|
|
624
|
+
targetDefIndices(target) {
|
|
625
|
+
if (target.type === 'identifier')
|
|
626
|
+
return target.text === '_' ? [] : [this.resolve(target)];
|
|
627
|
+
if (PATTERN_LIST_TYPES.has(target.type)) {
|
|
628
|
+
const out = [];
|
|
629
|
+
for (let i = 0; i < target.namedChildCount; i++) {
|
|
630
|
+
const c = target.namedChild(i);
|
|
631
|
+
if (c)
|
|
632
|
+
out.push(...this.targetDefIndices(c));
|
|
633
|
+
}
|
|
634
|
+
return out;
|
|
627
635
|
}
|
|
628
|
-
if (
|
|
629
|
-
|
|
636
|
+
if (target.type === 'list_splat_pattern') {
|
|
637
|
+
const id = target.namedChild(0);
|
|
638
|
+
return id ? this.targetDefIndices(id) : [];
|
|
639
|
+
}
|
|
640
|
+
return [];
|
|
641
|
+
}
|
|
642
|
+
/** Record result defs for a call-valued assignment. */
|
|
643
|
+
registerResultDefs(value, defs) {
|
|
644
|
+
if (defs.length === 0)
|
|
645
|
+
return;
|
|
646
|
+
const root = this.unwrapValue(value);
|
|
647
|
+
if (root.type === 'call')
|
|
648
|
+
this.resultDefTargets.set(root.id, [...defs]);
|
|
630
649
|
}
|
|
650
|
+
// ── taint-site harvest (#2227 follow-up) ─────────────────────────────────
|
|
631
651
|
/** Strip `parenthesized_expression` wrappers around a value (`(f())`). */
|
|
632
652
|
unwrapValue(node) {
|
|
633
653
|
let n = node;
|
|
@@ -681,31 +701,44 @@ export class PythonHarvester {
|
|
|
681
701
|
const resultDefs = this.resultDefTargets.get(node.id);
|
|
682
702
|
if (resultDefs !== undefined)
|
|
683
703
|
acc.setSiteResultDefs(siteIdx, resultDefs);
|
|
684
|
-
|
|
685
|
-
|
|
686
|
-
|
|
687
|
-
|
|
688
|
-
|
|
689
|
-
|
|
690
|
-
|
|
691
|
-
|
|
692
|
-
|
|
693
|
-
|
|
694
|
-
|
|
695
|
-
|
|
696
|
-
|
|
697
|
-
|
|
698
|
-
|
|
699
|
-
|
|
700
|
-
|
|
701
|
-
|
|
702
|
-
|
|
703
|
-
|
|
704
|
-
|
|
704
|
+
this.walkArgs(argsNode, acc, siteIdx);
|
|
705
|
+
acc.popFrame();
|
|
706
|
+
}
|
|
707
|
+
/** Walk arguments, assigning only positional values to positional sink slots. */
|
|
708
|
+
walkArgs(args, acc, siteIdx) {
|
|
709
|
+
if (!args)
|
|
710
|
+
return;
|
|
711
|
+
let pos = 0;
|
|
712
|
+
for (let i = 0; i < args.namedChildCount; i++) {
|
|
713
|
+
const arg = args.namedChild(i);
|
|
714
|
+
if (!arg || arg.type === 'comment')
|
|
715
|
+
continue;
|
|
716
|
+
if (arg.type === 'keyword_argument') {
|
|
717
|
+
const value = arg.childForFieldName('value');
|
|
718
|
+
// SiteRecord has no keyword-name metadata. Keep the value's ordinary
|
|
719
|
+
// uses/sources, but do not guess a positional sink slot from source order.
|
|
720
|
+
if (value)
|
|
721
|
+
acc.suppressOccurrences(() => this.walkValue(value, acc));
|
|
722
|
+
}
|
|
723
|
+
else if (arg.type === 'list_splat') {
|
|
724
|
+
acc.setFrameArg(pos);
|
|
725
|
+
acc.setSiteSpread(siteIdx, pos);
|
|
726
|
+
const inner = arg.namedChild(0);
|
|
727
|
+
if (inner)
|
|
728
|
+
this.walkValue(inner, acc);
|
|
729
|
+
pos++;
|
|
730
|
+
}
|
|
731
|
+
else if (arg.type === 'dictionary_splat') {
|
|
732
|
+
const inner = arg.namedChild(0);
|
|
733
|
+
if (inner)
|
|
734
|
+
acc.suppressOccurrences(() => this.walkValue(inner, acc));
|
|
735
|
+
}
|
|
736
|
+
else {
|
|
737
|
+
acc.setFrameArg(pos);
|
|
738
|
+
this.walkValue(arg, acc);
|
|
705
739
|
pos++;
|
|
706
740
|
}
|
|
707
741
|
}
|
|
708
|
-
acc.popFrame();
|
|
709
742
|
}
|
|
710
743
|
/**
|
|
711
744
|
* `attribute` chain walk shared by value position and callee position. Records
|
|
@@ -520,11 +520,11 @@ export function runScopeResolution(input, provider) {
|
|
|
520
520
|
// ── M3 taint setup (#2083 U4) ────────────────────────────────────────
|
|
521
521
|
// Explicit model-registration seam (idempotent, cheap) — the registry
|
|
522
522
|
// stays empty on non-pdg runs, preserving default-run parity. The
|
|
523
|
-
// registry is keyed by
|
|
524
|
-
//
|
|
525
|
-
//
|
|
526
|
-
//
|
|
527
|
-
// (
|
|
523
|
+
// registry is keyed by SupportedLanguages enum values, and
|
|
524
|
+
// ScopeResolver.language is registered under those same constants -
|
|
525
|
+
// the join is direct equality, with no mapping table. A language without a
|
|
526
|
+
// registered spec (go, ruby, ...) skips taint entirely: no work, no warn spam
|
|
527
|
+
// (KTD8).
|
|
528
528
|
registerBuiltinTaintModels();
|
|
529
529
|
const taintSpec = getSourceSinkConfig(provider.language);
|
|
530
530
|
// Taint-side solver fact cap: the SAME derivation emitFileReachingDefs
|
|
@@ -0,0 +1,10 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Built-in Python taint model (#2204 first slice).
|
|
3
|
+
*
|
|
4
|
+
* Keep the model intentionally conservative: import-aware sinks for standard
|
|
5
|
+
* library command execution, receiver-conventional database execution calls,
|
|
6
|
+
* and Flask/FastAPI-style request-object member reads. No sanitizers are
|
|
7
|
+
* registered yet because a false sanitizer kill can hide a real finding.
|
|
8
|
+
*/
|
|
9
|
+
import type { SourceSinkSanitizerSpec } from './source-sink-config.js';
|
|
10
|
+
export declare const PYTHON_TAINT_MODEL: SourceSinkSanitizerSpec;
|
|
@@ -0,0 +1,43 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Built-in Python taint model (#2204 first slice).
|
|
3
|
+
*
|
|
4
|
+
* Keep the model intentionally conservative: import-aware sinks for standard
|
|
5
|
+
* library command execution, receiver-conventional database execution calls,
|
|
6
|
+
* and Flask/FastAPI-style request-object member reads. No sanitizers are
|
|
7
|
+
* registered yet because a false sanitizer kill can hide a real finding.
|
|
8
|
+
*/
|
|
9
|
+
export const PYTHON_TAINT_MODEL = {
|
|
10
|
+
sources: [
|
|
11
|
+
{
|
|
12
|
+
kind: 'remote-input',
|
|
13
|
+
objects: ['request', 'req'],
|
|
14
|
+
properties: [
|
|
15
|
+
'args',
|
|
16
|
+
'form',
|
|
17
|
+
'json',
|
|
18
|
+
'data',
|
|
19
|
+
'headers',
|
|
20
|
+
'cookies',
|
|
21
|
+
'path_params',
|
|
22
|
+
'query_params',
|
|
23
|
+
],
|
|
24
|
+
},
|
|
25
|
+
],
|
|
26
|
+
sinks: [
|
|
27
|
+
{ name: 'system', kind: 'command-injection', args: [0], module: 'os' },
|
|
28
|
+
{ name: 'popen', kind: 'command-injection', args: [0], module: 'os' },
|
|
29
|
+
{ name: 'call', kind: 'command-injection', args: [0], module: 'subprocess' },
|
|
30
|
+
{ name: 'run', kind: 'command-injection', args: [0], module: 'subprocess' },
|
|
31
|
+
{ name: 'Popen', kind: 'command-injection', args: [0], module: 'subprocess' },
|
|
32
|
+
{ name: 'check_call', kind: 'command-injection', args: [0], module: 'subprocess' },
|
|
33
|
+
{ name: 'check_output', kind: 'command-injection', args: [0], module: 'subprocess' },
|
|
34
|
+
{ name: 'eval', kind: 'code-injection', args: [0], global: true },
|
|
35
|
+
{ name: 'exec', kind: 'code-injection', args: [0], global: true },
|
|
36
|
+
{ name: 'open', kind: 'path-traversal', args: [0], global: true },
|
|
37
|
+
{ name: 'query', kind: 'sql-injection', args: [0], anyReceiver: true },
|
|
38
|
+
{ name: 'execute', kind: 'sql-injection', args: [0], anyReceiver: true },
|
|
39
|
+
{ name: 'executemany', kind: 'sql-injection', args: [0], anyReceiver: true },
|
|
40
|
+
{ name: 'executescript', kind: 'sql-injection', args: [0], anyReceiver: true },
|
|
41
|
+
],
|
|
42
|
+
sanitizers: [],
|
|
43
|
+
};
|
|
@@ -27,12 +27,20 @@ export declare const TS_JS_TAINT_MODEL: SourceSinkSanitizerSpec;
|
|
|
27
27
|
* array order is semantic (entry identity) and intentionally preserved.
|
|
28
28
|
*/
|
|
29
29
|
export declare function computeTaintModelVersion(spec: SourceSinkSanitizerSpec): string;
|
|
30
|
-
|
|
30
|
+
export declare const BUILTIN_TAINT_MODELS: {
|
|
31
|
+
readonly javascript: SourceSinkSanitizerSpec;
|
|
32
|
+
readonly python: SourceSinkSanitizerSpec;
|
|
33
|
+
readonly typescript: SourceSinkSanitizerSpec;
|
|
34
|
+
};
|
|
35
|
+
/**
|
|
36
|
+
* Version stamp of every built-in model (joins the RepoMeta pdg key in U5).
|
|
37
|
+
* Adding a language model must invalidate existing persisted taint findings.
|
|
38
|
+
*/
|
|
31
39
|
export declare const taintModelVersion: string;
|
|
32
40
|
/**
|
|
33
|
-
* Register the built-in
|
|
34
|
-
* seam for the U4 emit path (call before the pdg window
|
|
35
|
-
* registry); idempotent.
|
|
36
|
-
*
|
|
41
|
+
* Register the built-in models for TypeScript, JavaScript, and Python.
|
|
42
|
+
* Explicit init seam for the U4 emit path (call before the pdg window
|
|
43
|
+
* consumes the registry); idempotent. Other language ids remain unregistered
|
|
44
|
+
* until they have a dedicated model.
|
|
37
45
|
*/
|
|
38
46
|
export declare function registerBuiltinTaintModels(): void;
|
|
@@ -16,6 +16,7 @@
|
|
|
16
16
|
*/
|
|
17
17
|
import { createHash } from 'node:crypto';
|
|
18
18
|
import { SupportedLanguages } from '../../../_shared/index.js';
|
|
19
|
+
import { PYTHON_TAINT_MODEL } from './python-model.js';
|
|
19
20
|
import { registerSourceSinkConfig } from './source-sink-registry.js';
|
|
20
21
|
/**
|
|
21
22
|
* The built-in TS/JS model. Module provenance uses bare specifier names —
|
|
@@ -74,7 +75,10 @@ export const TS_JS_TAINT_MODEL = {
|
|
|
74
75
|
* array order is semantic (entry identity) and intentionally preserved.
|
|
75
76
|
*/
|
|
76
77
|
export function computeTaintModelVersion(spec) {
|
|
77
|
-
return
|
|
78
|
+
return computeModelDigest(spec);
|
|
79
|
+
}
|
|
80
|
+
function computeModelDigest(value) {
|
|
81
|
+
return createHash('sha256').update(canonicalJson(value)).digest('hex').slice(0, 12);
|
|
78
82
|
}
|
|
79
83
|
function canonicalJson(value) {
|
|
80
84
|
if (Array.isArray(value))
|
|
@@ -88,15 +92,24 @@ function canonicalJson(value) {
|
|
|
88
92
|
}
|
|
89
93
|
return JSON.stringify(value);
|
|
90
94
|
}
|
|
91
|
-
|
|
92
|
-
|
|
95
|
+
export const BUILTIN_TAINT_MODELS = {
|
|
96
|
+
[SupportedLanguages.JavaScript]: TS_JS_TAINT_MODEL,
|
|
97
|
+
[SupportedLanguages.Python]: PYTHON_TAINT_MODEL,
|
|
98
|
+
[SupportedLanguages.TypeScript]: TS_JS_TAINT_MODEL,
|
|
99
|
+
};
|
|
100
|
+
/**
|
|
101
|
+
* Version stamp of every built-in model (joins the RepoMeta pdg key in U5).
|
|
102
|
+
* Adding a language model must invalidate existing persisted taint findings.
|
|
103
|
+
*/
|
|
104
|
+
export const taintModelVersion = computeModelDigest(BUILTIN_TAINT_MODELS);
|
|
93
105
|
/**
|
|
94
|
-
* Register the built-in
|
|
95
|
-
* seam for the U4 emit path (call before the pdg window
|
|
96
|
-
* registry); idempotent.
|
|
97
|
-
*
|
|
106
|
+
* Register the built-in models for TypeScript, JavaScript, and Python.
|
|
107
|
+
* Explicit init seam for the U4 emit path (call before the pdg window
|
|
108
|
+
* consumes the registry); idempotent. Other language ids remain unregistered
|
|
109
|
+
* until they have a dedicated model.
|
|
98
110
|
*/
|
|
99
111
|
export function registerBuiltinTaintModels() {
|
|
100
112
|
registerSourceSinkConfig(SupportedLanguages.TypeScript, TS_JS_TAINT_MODEL);
|
|
101
113
|
registerSourceSinkConfig(SupportedLanguages.JavaScript, TS_JS_TAINT_MODEL);
|
|
114
|
+
registerSourceSinkConfig(SupportedLanguages.Python, PYTHON_TAINT_MODEL);
|
|
102
115
|
}
|
|
@@ -583,8 +583,11 @@ const processBatch = (files, onProgress) => {
|
|
|
583
583
|
setLanguage(language, regularFiles[0].path);
|
|
584
584
|
processFileGroup(regularFiles, language, queryString, result, onFileProcessed);
|
|
585
585
|
}
|
|
586
|
-
catch {
|
|
587
|
-
//
|
|
586
|
+
catch (err) {
|
|
587
|
+
// A throw here drops the whole language group — surface it to the pool
|
|
588
|
+
// (#2264) instead of silently skipping. The old empty catch hid real
|
|
589
|
+
// extractor/parser failures, not just an unavailable grammar.
|
|
590
|
+
reportWarning(`Skipped ${regularFiles.length} ${language} file(s) after a processing error: ${err instanceof Error ? err.message : String(err)}`);
|
|
588
591
|
}
|
|
589
592
|
}
|
|
590
593
|
else {
|
|
@@ -599,8 +602,10 @@ const processBatch = (files, onProgress) => {
|
|
|
599
602
|
setLanguage(language, tsxFiles[0].path);
|
|
600
603
|
processFileGroup(tsxFiles, language, queryString, result, onFileProcessed);
|
|
601
604
|
}
|
|
602
|
-
catch {
|
|
603
|
-
//
|
|
605
|
+
catch (err) {
|
|
606
|
+
// See above — surface a tsx-group processing failure rather than
|
|
607
|
+
// silently dropping every file in it (#2264).
|
|
608
|
+
reportWarning(`Skipped ${tsxFiles.length} ${language} (tsx) file(s) after a processing error: ${err instanceof Error ? err.message : String(err)}`);
|
|
604
609
|
}
|
|
605
610
|
}
|
|
606
611
|
else {
|
|
@@ -745,6 +750,23 @@ export function extractORMQueries(filePath, content, out) {
|
|
|
745
750
|
// per file; this module does not re-export it. Downstream consumers
|
|
746
751
|
// import the function and its types directly from `route-extractors/`.
|
|
747
752
|
import { extractFastAPIRouterBindings } from '../route-extractors/fastapi-router-bindings.js';
|
|
753
|
+
/**
|
|
754
|
+
* Report a non-fatal worker issue to the pool over IPC so a caught error is not
|
|
755
|
+
* invisible to the operator (#2264). The pool logs it on the main thread AND
|
|
756
|
+
* resets the worker idle timer (so a worker grinding through failing files isn't
|
|
757
|
+
* falsely idle-evicted). Falls back to the local logger when there's no parent —
|
|
758
|
+
* this code also runs on the main thread in tests / the non-worker path. Fatal,
|
|
759
|
+
* group-aborting errors go through the message handler's
|
|
760
|
+
* `{ type: 'error', errorStack }` channel instead.
|
|
761
|
+
*/
|
|
762
|
+
function reportWarning(message) {
|
|
763
|
+
if (parentPort) {
|
|
764
|
+
parentPort.postMessage({ type: 'warning', message });
|
|
765
|
+
}
|
|
766
|
+
else {
|
|
767
|
+
logger.warn(message);
|
|
768
|
+
}
|
|
769
|
+
}
|
|
748
770
|
const processFileGroup = (files, language, queryString, result, onFileProcessed) => {
|
|
749
771
|
let query;
|
|
750
772
|
try {
|
|
@@ -752,13 +774,7 @@ const processFileGroup = (files, language, queryString, result, onFileProcessed)
|
|
|
752
774
|
query = new Parser.Query(lang, queryString);
|
|
753
775
|
}
|
|
754
776
|
catch (err) {
|
|
755
|
-
|
|
756
|
-
if (parentPort) {
|
|
757
|
-
parentPort.postMessage({ type: 'warning', message });
|
|
758
|
-
}
|
|
759
|
-
else {
|
|
760
|
-
logger.warn(message);
|
|
761
|
-
}
|
|
777
|
+
reportWarning(`Query compilation failed for ${language}: ${err instanceof Error ? err.message : String(err)}`);
|
|
762
778
|
return;
|
|
763
779
|
}
|
|
764
780
|
for (const file of files) {
|
|
@@ -799,7 +815,7 @@ const processFileGroup = (files, language, queryString, result, onFileProcessed)
|
|
|
799
815
|
});
|
|
800
816
|
}
|
|
801
817
|
catch (err) {
|
|
802
|
-
|
|
818
|
+
reportWarning(`Failed to parse file ${file.path}: ${err instanceof Error ? err.message : String(err)}`);
|
|
803
819
|
continue;
|
|
804
820
|
}
|
|
805
821
|
result.fileCount++;
|
|
@@ -809,7 +825,7 @@ const processFileGroup = (files, language, queryString, result, onFileProcessed)
|
|
|
809
825
|
matches = query.matches(tree.rootNode);
|
|
810
826
|
}
|
|
811
827
|
catch (err) {
|
|
812
|
-
|
|
828
|
+
reportWarning(`Query execution failed for ${file.path}: ${err instanceof Error ? err.message : String(err)}`);
|
|
813
829
|
continue;
|
|
814
830
|
}
|
|
815
831
|
const concreteTypedefRanges = buildConcreteTypedefDefinitionRanges(matches);
|
|
@@ -822,14 +838,7 @@ const processFileGroup = (files, language, queryString, result, onFileProcessed)
|
|
|
822
838
|
// see parsedfile-store.ts). parse-impl flushes `result.parsedFiles` to disk
|
|
823
839
|
// per chunk and does NOT retain them in main-thread heap, so this no longer
|
|
824
840
|
// costs ~1× the semantic model in RAM during parse.
|
|
825
|
-
const parsedFile = extractParsedFile(provider, parseContent, file.path,
|
|
826
|
-
if (parentPort) {
|
|
827
|
-
parentPort.postMessage({ type: 'warning', message });
|
|
828
|
-
}
|
|
829
|
-
else {
|
|
830
|
-
logger.warn(message);
|
|
831
|
-
}
|
|
832
|
-
}, tree, scopeSourceKind);
|
|
841
|
+
const parsedFile = extractParsedFile(provider, parseContent, file.path, reportWarning, tree, scopeSourceKind);
|
|
833
842
|
if (parsedFile !== undefined) {
|
|
834
843
|
// Capture-time side-channel (#1983): `extractParsedFile` just ran the
|
|
835
844
|
// provider's `emitScopeCaptures`, which (for C++ ADL/namespace marks,
|
|
@@ -884,11 +893,7 @@ const processFileGroup = (files, language, queryString, result, onFileProcessed)
|
|
|
884
893
|
}
|
|
885
894
|
}
|
|
886
895
|
catch (err) {
|
|
887
|
-
|
|
888
|
-
if (parentPort)
|
|
889
|
-
parentPort.postMessage({ type: 'warning', message });
|
|
890
|
-
else
|
|
891
|
-
logger.warn(message);
|
|
896
|
+
reportWarning(`CFG build failed for ${file.path}: ${err instanceof Error ? err.message : String(err)}`);
|
|
892
897
|
}
|
|
893
898
|
}
|
|
894
899
|
result.parsedFiles.push(withChannels);
|
|
@@ -1575,7 +1580,10 @@ const processFileGroup = (files, language, queryString, result, onFileProcessed)
|
|
|
1575
1580
|
constraintsTag = templateConstraintsIdTag(parsedTemplateConstraints);
|
|
1576
1581
|
}
|
|
1577
1582
|
}
|
|
1578
|
-
catch {
|
|
1583
|
+
catch (err) {
|
|
1584
|
+
// Optional C++ template-constraint enrichment: fall back to no tag, but
|
|
1585
|
+
// surface the failure (#2264) — matches the CFG-build warning above.
|
|
1586
|
+
reportWarning(`Template-constraint extraction failed for ${file.path}: ${err instanceof Error ? err.message : String(err)}`);
|
|
1579
1587
|
parsedTemplateConstraints = undefined;
|
|
1580
1588
|
constraintsTag = '';
|
|
1581
1589
|
}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
export declare const withConnLock: <T>(fn: () => Promise<T>) => Promise<T>;
|
|
@@ -0,0 +1,61 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Serialize every operation on the shared singleton LadybugDB connection.
|
|
3
|
+
*
|
|
4
|
+
* LadybugDB is single-writer and its `Connection` is NOT safe for concurrent
|
|
5
|
+
* query execution: dispatching two queries on one connection at the same time
|
|
6
|
+
* lets two libuv workers mutate shared native engine state at once, corrupting
|
|
7
|
+
* the heap. This surfaced as `double free or corruption (out)` / SIGSEGV at the
|
|
8
|
+
* end of `analyze --pdg`, where the periodic WAL-checkpoint driver
|
|
9
|
+
* (`wal-checkpoint-driver.ts`) fired `CHECKPOINT` on the same connection a
|
|
10
|
+
* long-running PDG-table COPY was still using. `--pdg` makes those COPYs outlast
|
|
11
|
+
* the driver's 5 s tick, so the overlap (rare without `--pdg`) becomes reliable.
|
|
12
|
+
*
|
|
13
|
+
* Every singleton-`conn` helper in `lbug-adapter.ts` runs its full query +
|
|
14
|
+
* result-drain inside this lock, so the checkpoint driver, the bulk COPY, the
|
|
15
|
+
* embedding writeback, and the PDG edge deletes are mutually exclusive — the
|
|
16
|
+
* property that makes a strictly-serial workload stable.
|
|
17
|
+
*
|
|
18
|
+
* Implementation: a promise chain. Each caller installs a fresh unresolved tail,
|
|
19
|
+
* awaits the previous holder's tail, runs, then releases its own in `finally`
|
|
20
|
+
* (so a thrown op never wedges the connection). FIFO and non-reentrant: a wrapped
|
|
21
|
+
* helper MUST NOT call another wrapped helper — the inner call would await its own
|
|
22
|
+
* holder's tail and deadlock. The re-entry guard below catches this and throws
|
|
23
|
+
* instead of hanging. A boolean flag can't do this: a legitimately-queued
|
|
24
|
+
* top-level caller also runs while the lock is held, so only AsyncLocalStorage —
|
|
25
|
+
* which marks the *async context* of the running `fn` — distinguishes a true
|
|
26
|
+
* nested call from normal contention.
|
|
27
|
+
*/
|
|
28
|
+
import { AsyncLocalStorage } from 'node:async_hooks';
|
|
29
|
+
let tail = Promise.resolve();
|
|
30
|
+
// Set (to `true`) only inside a holding `fn`'s async context. A withConnLock call
|
|
31
|
+
// that observes it set is a nested/re-entrant call from within a critical section.
|
|
32
|
+
const inCriticalSection = new AsyncLocalStorage();
|
|
33
|
+
export const withConnLock = async (fn) => {
|
|
34
|
+
if (inCriticalSection.getStore()) {
|
|
35
|
+
throw new Error('conn-lock re-entry: a withConnLock-wrapped helper called another wrapped ' +
|
|
36
|
+
'helper, which would deadlock the single LadybugDB connection. Run the inner ' +
|
|
37
|
+
'work outside the lock, or inline it. See src/core/lbug/conn-lock.ts.');
|
|
38
|
+
}
|
|
39
|
+
const prior = tail;
|
|
40
|
+
let release;
|
|
41
|
+
tail = new Promise((resolve) => {
|
|
42
|
+
release = resolve;
|
|
43
|
+
});
|
|
44
|
+
await prior;
|
|
45
|
+
try {
|
|
46
|
+
return await inCriticalSection.run(true, fn);
|
|
47
|
+
}
|
|
48
|
+
finally {
|
|
49
|
+
release();
|
|
50
|
+
}
|
|
51
|
+
};
|
|
52
|
+
/**
|
|
53
|
+
* Test-only: reset the lock chain to a fresh resolved tail. Production code has
|
|
54
|
+
* no reason to call this — a leaked-but-resolved tail is harmless — but unit
|
|
55
|
+
* tests want a clean chain per case.
|
|
56
|
+
*
|
|
57
|
+
* @internal
|
|
58
|
+
*/
|
|
59
|
+
export const _resetConnLockForTests = () => {
|
|
60
|
+
tail = Promise.resolve();
|
|
61
|
+
};
|