circle-ir-ai 2.71.0 → 2.76.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +194 -0
- package/dist/agents/mastra/workflow.d.ts +3 -0
- package/dist/agents/mastra/workflow.d.ts.map +1 -1
- package/dist/agents/mastra/workflow.js +46 -2
- package/dist/agents/mastra/workflow.js.map +1 -1
- package/dist/llm/verification.d.ts +18 -13
- package/dist/llm/verification.d.ts.map +1 -1
- package/dist/llm/verification.js +41 -27
- package/dist/llm/verification.js.map +1 -1
- package/dist/security-scan/scanner.d.ts +10 -0
- package/dist/security-scan/scanner.d.ts.map +1 -1
- package/dist/security-scan/scanner.js +35 -4
- package/dist/security-scan/scanner.js.map +1 -1
- package/dist/security-scan/sink-filters.d.ts +1 -1
- package/dist/security-scan/sink-filters.d.ts.map +1 -1
- package/dist/security-scan/sink-filters.js +81 -3
- package/dist/security-scan/sink-filters.js.map +1 -1
- package/dist/utils/file-skip.d.ts +81 -0
- package/dist/utils/file-skip.d.ts.map +1 -1
- package/dist/utils/file-skip.js +134 -0
- package/dist/utils/file-skip.js.map +1 -1
- package/dist/utils/scan-heartbeat.d.ts +52 -0
- package/dist/utils/scan-heartbeat.d.ts.map +1 -0
- package/dist/utils/scan-heartbeat.js +85 -0
- package/dist/utils/scan-heartbeat.js.map +1 -0
- package/package.json +1 -1
package/dist/utils/file-skip.js
CHANGED
|
@@ -48,6 +48,93 @@ export function isMinifiedFilename(filePath) {
|
|
|
48
48
|
const base = filePath.split(/[\\/]/).pop() ?? filePath;
|
|
49
49
|
return MINIFIED_FILENAME_RE.test(base);
|
|
50
50
|
}
|
|
51
|
+
/**
|
|
52
|
+
* cognium-ai#252 (problem C) — mirrored CDN/vendor trees.
|
|
53
|
+
*
|
|
54
|
+
* #243 catches build artifacts by the `.min.` convention, but mirrored CDN
|
|
55
|
+
* copies frequently carry neither `.min.` nor a giant line: site-rippers
|
|
56
|
+
* (wget/httrack and friends) flatten a remote URL into a filename by
|
|
57
|
+
* replacing separators, producing names like
|
|
58
|
+
*
|
|
59
|
+
* js/ajax.googleapis.com__ajax__libs__dojo__1.4.3__dojo__dojo.js
|
|
60
|
+
*
|
|
61
|
+
* poisontap ships 249 such copies. They are third-party code the repo owner
|
|
62
|
+
* does not maintain, so findings in them are noise, and in #252 they were the
|
|
63
|
+
* trees that walked discovery into a 7200s timeout with no artifact.
|
|
64
|
+
*
|
|
65
|
+
* Two conservative signals, either sufficient:
|
|
66
|
+
*
|
|
67
|
+
* 1. a hostname-shaped token ending in a real TLD, immediately followed by
|
|
68
|
+
* the `__` flattening marker — `ajax.googleapis.com__`. Both conditions
|
|
69
|
+
* matter: the marker keeps ordinary dotted names (`utils.test.js`,
|
|
70
|
+
* `app.config.js`) out, and the TLD list keeps first-party names that
|
|
71
|
+
* happen to contain `__` (`foo.bar__baz.js`) out.
|
|
72
|
+
* 2. a known CDN host appearing anywhere in the path, for mirrors that keep
|
|
73
|
+
* real directories (`vendor/cdnjs.cloudflare.com/ajax/libs/...`).
|
|
74
|
+
*
|
|
75
|
+
* Opt out with `SKIP_CDN_MIRRORS=0`.
|
|
76
|
+
*/
|
|
77
|
+
export const CDN_MIRROR_FLATTENED_RE = /(?:[a-z0-9-]+\.)+(?:com|net|org|io|dev|co|edu|gov|info|biz|app|cloud|cn|jp|uk|de|fr|ru|nl|eu|ca|au|br|in)__/i;
|
|
78
|
+
/** Hosts common enough to be worth matching without the `__` marker. */
|
|
79
|
+
export const KNOWN_CDN_HOSTS = [
|
|
80
|
+
'ajax.googleapis.com',
|
|
81
|
+
'cdnjs.cloudflare.com',
|
|
82
|
+
'cdn.jsdelivr.net',
|
|
83
|
+
'unpkg.com',
|
|
84
|
+
'code.jquery.com',
|
|
85
|
+
'maxcdn.bootstrapcdn.com',
|
|
86
|
+
'stackpath.bootstrapcdn.com',
|
|
87
|
+
'ajax.aspnetcdn.com',
|
|
88
|
+
'ajax.microsoft.com',
|
|
89
|
+
];
|
|
90
|
+
/**
|
|
91
|
+
* cognium-ai#251 — vendored third-party trees.
|
|
92
|
+
*
|
|
93
|
+
* #243 (`.min.` names) and #252 (CDN mirrors) both key on the FILENAME. The
|
|
94
|
+
* remaining holdouts were vendored *bundles* that look like ordinary source by
|
|
95
|
+
* name and pass every content cap: `prettier/lib/parser-*.js` sits under the
|
|
96
|
+
* 1 MB size cap and the 50k line cap, yet wedges analysis at ~100% CPU for
|
|
97
|
+
* hours. #251 reported 0/45 repos completing because of it.
|
|
98
|
+
*
|
|
99
|
+
* The durable signal is not the file, it is the directory: code under
|
|
100
|
+
* `node_modules/`, `vendor/`, `third_party/` or `bower_components/` is not
|
|
101
|
+
* maintained by the repo owner, so findings in it are noise regardless of
|
|
102
|
+
* whether it happens to parse quickly.
|
|
103
|
+
*
|
|
104
|
+
* Note the CPU wedge itself is separately bounded since 2.66.0 — with
|
|
105
|
+
* `SCAN_PARSE_ISOLATION=1` a pathological parse aborts at the per-file
|
|
106
|
+
* timeout instead of holding the scan. This skip is about not spending the
|
|
107
|
+
* time at all, and about not reporting third-party findings as the repo's.
|
|
108
|
+
*
|
|
109
|
+
* Opt out with `SKIP_VENDORED=0`.
|
|
110
|
+
*/
|
|
111
|
+
export const VENDORED_DIR_RE = /(?:^|\/)(?:node_modules|bower_components|vendor|vendored|third_party|thirdparty|external)(?:\/|$)/i;
|
|
112
|
+
/**
|
|
113
|
+
* Generated parser bundles that ship inside a library's own `lib/` directory
|
|
114
|
+
* — the specific shape #251 reported. Anchored on the `parser-` prefix AND a
|
|
115
|
+
* `lib/` parent so a first-party `src/parser-json.js` is untouched.
|
|
116
|
+
*/
|
|
117
|
+
export const VENDORED_BUNDLE_RE = /(?:^|\/)lib\/parser-[\w.-]+\.(?:js|mjs|cjs)$/i;
|
|
118
|
+
/** cognium-ai#251 — true when the path is third-party vendored code. */
|
|
119
|
+
export function isVendoredPath(filePath) {
|
|
120
|
+
if (process.env.SKIP_VENDORED === '0')
|
|
121
|
+
return false;
|
|
122
|
+
const normalized = filePath.replace(/\\/g, '/');
|
|
123
|
+
return VENDORED_DIR_RE.test(normalized) || VENDORED_BUNDLE_RE.test(normalized);
|
|
124
|
+
}
|
|
125
|
+
/**
|
|
126
|
+
* cognium-ai#252 — true when the path looks like a mirrored copy of a
|
|
127
|
+
* third-party CDN asset rather than first-party source.
|
|
128
|
+
*/
|
|
129
|
+
export function isCdnMirrorPath(filePath) {
|
|
130
|
+
if (process.env.SKIP_CDN_MIRRORS === '0')
|
|
131
|
+
return false;
|
|
132
|
+
const normalized = filePath.replace(/\\/g, '/').toLowerCase();
|
|
133
|
+
const base = normalized.split('/').pop() ?? normalized;
|
|
134
|
+
if (CDN_MIRROR_FLATTENED_RE.test(base))
|
|
135
|
+
return true;
|
|
136
|
+
return KNOWN_CDN_HOSTS.some((h) => normalized.includes(h));
|
|
137
|
+
}
|
|
51
138
|
/**
|
|
52
139
|
* Pre-flight skip check for a single file. Reads content as a side
|
|
53
140
|
* effect so the caller doesn't double-read.
|
|
@@ -65,6 +152,22 @@ export function preFlightSkip(filePath) {
|
|
|
65
152
|
reason: `minified build artifact by filename (${filePath.split(/[\\/]/).pop()})`,
|
|
66
153
|
};
|
|
67
154
|
}
|
|
155
|
+
// cognium-ai#252 (problem C) — mirrored CDN/vendor trees. Cheap (no stat,
|
|
156
|
+
// no read) and refuses third-party copies the repo owner does not maintain
|
|
157
|
+
// before they reach the parser or the LLM.
|
|
158
|
+
if (isCdnMirrorPath(filePath)) {
|
|
159
|
+
return {
|
|
160
|
+
skip: true,
|
|
161
|
+
reason: `mirrored CDN/vendor asset (${filePath.split(/[\\/]/).pop()})`,
|
|
162
|
+
};
|
|
163
|
+
}
|
|
164
|
+
// cognium-ai#251 — vendored third-party trees and generated parser bundles.
|
|
165
|
+
if (isVendoredPath(filePath)) {
|
|
166
|
+
return {
|
|
167
|
+
skip: true,
|
|
168
|
+
reason: `vendored third-party code (${filePath.split(/[\\/]/).pop()})`,
|
|
169
|
+
};
|
|
170
|
+
}
|
|
68
171
|
let stat;
|
|
69
172
|
try {
|
|
70
173
|
stat = fs.statSync(filePath);
|
|
@@ -109,4 +212,35 @@ export function preFlightSkip(filePath) {
|
|
|
109
212
|
}
|
|
110
213
|
return { skip: false, content, size: stat.size };
|
|
111
214
|
}
|
|
215
|
+
/**
|
|
216
|
+
* Detect minified / bundled single-line content. #102, relocated here for #252 problem D.
|
|
217
|
+
*
|
|
218
|
+
* Windowed verification can't help when every source/sink lives on the
|
|
219
|
+
* same logical line — the merged window degenerates to the whole file
|
|
220
|
+
* and recursive splitting can't reduce the body further. Skipping LLM
|
|
221
|
+
* verification on minified content avoids pure-cost-no-benefit LLM
|
|
222
|
+
* fan-out; static-analysis findings are unaffected.
|
|
223
|
+
*
|
|
224
|
+
* All three conditions must hold (conservative — avoids flagging
|
|
225
|
+
* legitimate files with one long embedded JSON / base64 line):
|
|
226
|
+
*
|
|
227
|
+
* - File > 50 KB
|
|
228
|
+
* - Longest line > 5,000 chars
|
|
229
|
+
* - Average line length > 500 chars
|
|
230
|
+
*/
|
|
231
|
+
export function isMinifiedContent(code) {
|
|
232
|
+
if (code.length < 50_000)
|
|
233
|
+
return false;
|
|
234
|
+
const lines = code.split(/\r?\n/);
|
|
235
|
+
if (lines.length === 0)
|
|
236
|
+
return false;
|
|
237
|
+
let maxLine = 0;
|
|
238
|
+
for (const l of lines)
|
|
239
|
+
if (l.length > maxLine)
|
|
240
|
+
maxLine = l.length;
|
|
241
|
+
if (maxLine < 5_000)
|
|
242
|
+
return false;
|
|
243
|
+
const avgLine = code.length / lines.length;
|
|
244
|
+
return avgLine > 500;
|
|
245
|
+
}
|
|
112
246
|
//# sourceMappingURL=file-skip.js.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"file-skip.js","sourceRoot":"","sources":["../../src/utils/file-skip.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;GAwBG;AAEH,OAAO,KAAK,EAAE,MAAM,IAAI,CAAC;AAazB,MAAM,cAAc,GAAG,GAAG,EAAE,CAAC,QAAQ,CAAC,OAAO,CAAC,GAAG,CAAC,cAAc,IAAI,SAAS,EAAE,EAAE,CAAC,CAAC;AACnF,MAAM,eAAe,GAAG,GAAG,EAAE,CAAC,QAAQ,CAAC,OAAO,CAAC,GAAG,CAAC,eAAe,IAAI,OAAO,EAAE,EAAE,CAAC,CAAC;AACnF,MAAM,cAAc,GAAG,GAAG,EAAE,CAAC,QAAQ,CAAC,OAAO,CAAC,GAAG,CAAC,cAAc,IAAI,OAAO,EAAE,EAAE,CAAC,CAAC;AAEjF;;;;;;;;;;;;;;GAcG;AACH,MAAM,CAAC,MAAM,oBAAoB,GAAG,0CAA0C,CAAC;AAE/E,kFAAkF;AAClF,MAAM,UAAU,kBAAkB,CAAC,QAAgB;IACjD,MAAM,IAAI,GAAG,QAAQ,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,GAAG,EAAE,IAAI,QAAQ,CAAC;IACvD,OAAO,oBAAoB,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;AACzC,CAAC;AAED;;;;;;GAMG;AACH,MAAM,UAAU,aAAa,CAAC,QAAgB;IAC5C,wEAAwE;IACxE,0EAA0E;IAC1E,0BAA0B;IAC1B,IAAI,kBAAkB,CAAC,QAAQ,CAAC,EAAE,CAAC;QACjC,OAAO;YACL,IAAI,EAAE,IAAI;YACV,MAAM,EAAE,wCAAwC,QAAQ,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,GAAG,EAAE,GAAG;SACjF,CAAC;IACJ,CAAC;IAED,IAAI,IAAc,CAAC;IACnB,IAAI,CAAC;QACH,IAAI,GAAG,EAAE,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC;IAC/B,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,EAAE,IAAI,EAAE,IAAI,EAAE,MAAM,EAAE,YAAY,EAAE,CAAC;IAC9C,CAAC;IAED,MAAM,OAAO,GAAG,cAAc,EAAE,CAAC;IACjC,IAAI,IAAI,CAAC,IAAI,GAAG,OAAO,EAAE,CAAC;QACxB,OAAO;YACL,IAAI,EAAE,IAAI;YACV,MAAM,EAAE,mBAAmB,CAAC,IAAI,CAAC,IAAI,GAAG,IAAI,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,SAAS,CAAC,OAAO,GAAG,IAAI,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,8BAA8B;YAC1H,IAAI,EAAE,IAAI,CAAC,IAAI;SAChB,CAAC;IACJ,CAAC;IAED,IAAI,OAAe,CAAC;IACpB,IAAI,CAAC;QACH,OAAO,GAAG,EAAE,CAAC,YAAY,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAC;IAC/C,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,EAAE,IAAI,EAAE,IAAI,EAAE,MAAM,EAAE,YAAY,EAAE,CAAC;IAC9C,CAAC;IAED,MAAM,KAAK,GAAG,OAAO,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;IAClC,MAAM,OAAO,GAAG,eAAe,EAAE,CAAC;IAClC,MAAM,aAAa,GAAG,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,CAAC,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC,CAAC;IACvE,IAAI,aAAa,GAAG,OAAO,EAAE,CAAC;QAC5B,OAAO;YACL,IAAI,EAAE,IAAI;YACV,MAAM,EAAE,kBAAkB,aAAa,YAAY,OAAO,oBAAoB;YAC9E,OAAO;YACP,IAAI,EAAE,IAAI,CAAC,IAAI;SAChB,CAAC;IACJ,CAAC;IAED,MAAM,QAAQ,GAAG,cAAc,EAAE,CAAC;IAClC,IAAI,KAAK,CAAC,MAAM,GAAG,QAAQ,EAAE,CAAC;QAC5B,OAAO;YACL,IAAI,EAAE,IAAI;YACV,MAAM,EAAE,mBAAmB,KAAK,CAAC,MAAM,MAAM,QAAQ,0BAA0B;YAC/E,OAAO;YACP,IAAI,EAAE,IAAI,CAAC,IAAI;SAChB,CAAC;IACJ,CAAC;IAED,OAAO,EAAE,IAAI,EAAE,KAAK,EAAE,OAAO,EAAE,IAAI,EAAE,IAAI,CAAC,IAAI,EAAE,CAAC;AACnD,CAAC"}
|
|
1
|
+
{"version":3,"file":"file-skip.js","sourceRoot":"","sources":["../../src/utils/file-skip.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;GAwBG;AAEH,OAAO,KAAK,EAAE,MAAM,IAAI,CAAC;AAazB,MAAM,cAAc,GAAG,GAAG,EAAE,CAAC,QAAQ,CAAC,OAAO,CAAC,GAAG,CAAC,cAAc,IAAI,SAAS,EAAE,EAAE,CAAC,CAAC;AACnF,MAAM,eAAe,GAAG,GAAG,EAAE,CAAC,QAAQ,CAAC,OAAO,CAAC,GAAG,CAAC,eAAe,IAAI,OAAO,EAAE,EAAE,CAAC,CAAC;AACnF,MAAM,cAAc,GAAG,GAAG,EAAE,CAAC,QAAQ,CAAC,OAAO,CAAC,GAAG,CAAC,cAAc,IAAI,OAAO,EAAE,EAAE,CAAC,CAAC;AAEjF;;;;;;;;;;;;;;GAcG;AACH,MAAM,CAAC,MAAM,oBAAoB,GAAG,0CAA0C,CAAC;AAE/E,kFAAkF;AAClF,MAAM,UAAU,kBAAkB,CAAC,QAAgB;IACjD,MAAM,IAAI,GAAG,QAAQ,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,GAAG,EAAE,IAAI,QAAQ,CAAC;IACvD,OAAO,oBAAoB,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;AACzC,CAAC;AAED;;;;;;;;;;;;;;;;;;;;;;;;;GAyBG;AACH,MAAM,CAAC,MAAM,uBAAuB,GAClC,8GAA8G,CAAC;AAEjH,wEAAwE;AACxE,MAAM,CAAC,MAAM,eAAe,GAAsB;IAChD,qBAAqB;IACrB,sBAAsB;IACtB,kBAAkB;IAClB,WAAW;IACX,iBAAiB;IACjB,yBAAyB;IACzB,4BAA4B;IAC5B,oBAAoB;IACpB,oBAAoB;CACrB,CAAC;AAEF;;;;;;;;;;;;;;;;;;;;GAoBG;AACH,MAAM,CAAC,MAAM,eAAe,GAC1B,oGAAoG,CAAC;AAEvG;;;;GAIG;AACH,MAAM,CAAC,MAAM,kBAAkB,GAAG,+CAA+C,CAAC;AAElF,wEAAwE;AACxE,MAAM,UAAU,cAAc,CAAC,QAAgB;IAC7C,IAAI,OAAO,CAAC,GAAG,CAAC,aAAa,KAAK,GAAG;QAAE,OAAO,KAAK,CAAC;IACpD,MAAM,UAAU,GAAG,QAAQ,CAAC,OAAO,CAAC,KAAK,EAAE,GAAG,CAAC,CAAC;IAChD,OAAO,eAAe,CAAC,IAAI,CAAC,UAAU,CAAC,IAAI,kBAAkB,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;AACjF,CAAC;AAED;;;GAGG;AACH,MAAM,UAAU,eAAe,CAAC,QAAgB;IAC9C,IAAI,OAAO,CAAC,GAAG,CAAC,gBAAgB,KAAK,GAAG;QAAE,OAAO,KAAK,CAAC;IACvD,MAAM,UAAU,GAAG,QAAQ,CAAC,OAAO,CAAC,KAAK,EAAE,GAAG,CAAC,CAAC,WAAW,EAAE,CAAC;IAC9D,MAAM,IAAI,GAAG,UAAU,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,IAAI,UAAU,CAAC;IACvD,IAAI,uBAAuB,CAAC,IAAI,CAAC,IAAI,CAAC;QAAE,OAAO,IAAI,CAAC;IACpD,OAAO,eAAe,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,UAAU,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAC;AAC7D,CAAC;AAED;;;;;;GAMG;AACH,MAAM,UAAU,aAAa,CAAC,QAAgB;IAC5C,wEAAwE;IACxE,0EAA0E;IAC1E,0BAA0B;IAC1B,IAAI,kBAAkB,CAAC,QAAQ,CAAC,EAAE,CAAC;QACjC,OAAO;YACL,IAAI,EAAE,IAAI;YACV,MAAM,EAAE,wCAAwC,QAAQ,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,GAAG,EAAE,GAAG;SACjF,CAAC;IACJ,CAAC;IAED,0EAA0E;IAC1E,2EAA2E;IAC3E,2CAA2C;IAC3C,IAAI,eAAe,CAAC,QAAQ,CAAC,EAAE,CAAC;QAC9B,OAAO;YACL,IAAI,EAAE,IAAI;YACV,MAAM,EAAE,8BAA8B,QAAQ,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,GAAG,EAAE,GAAG;SACvE,CAAC;IACJ,CAAC;IAED,4EAA4E;IAC5E,IAAI,cAAc,CAAC,QAAQ,CAAC,EAAE,CAAC;QAC7B,OAAO;YACL,IAAI,EAAE,IAAI;YACV,MAAM,EAAE,8BAA8B,QAAQ,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,GAAG,EAAE,GAAG;SACvE,CAAC;IACJ,CAAC;IAED,IAAI,IAAc,CAAC;IACnB,IAAI,CAAC;QACH,IAAI,GAAG,EAAE,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC;IAC/B,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,EAAE,IAAI,EAAE,IAAI,EAAE,MAAM,EAAE,YAAY,EAAE,CAAC;IAC9C,CAAC;IAED,MAAM,OAAO,GAAG,cAAc,EAAE,CAAC;IACjC,IAAI,IAAI,CAAC,IAAI,GAAG,OAAO,EAAE,CAAC;QACxB,OAAO;YACL,IAAI,EAAE,IAAI;YACV,MAAM,EAAE,mBAAmB,CAAC,IAAI,CAAC,IAAI,GAAG,IAAI,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,SAAS,CAAC,OAAO,GAAG,IAAI,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,8BAA8B;YAC1H,IAAI,EAAE,IAAI,CAAC,IAAI;SAChB,CAAC;IACJ,CAAC;IAED,IAAI,OAAe,CAAC;IACpB,IAAI,CAAC;QACH,OAAO,GAAG,EAAE,CAAC,YAAY,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAC;IAC/C,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,EAAE,IAAI,EAAE,IAAI,EAAE,MAAM,EAAE,YAAY,EAAE,CAAC;IAC9C,CAAC;IAED,MAAM,KAAK,GAAG,OAAO,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;IAClC,MAAM,OAAO,GAAG,eAAe,EAAE,CAAC;IAClC,MAAM,aAAa,GAAG,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,CAAC,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC,CAAC;IACvE,IAAI,aAAa,GAAG,OAAO,EAAE,CAAC;QAC5B,OAAO;YACL,IAAI,EAAE,IAAI;YACV,MAAM,EAAE,kBAAkB,aAAa,YAAY,OAAO,oBAAoB;YAC9E,OAAO;YACP,IAAI,EAAE,IAAI,CAAC,IAAI;SAChB,CAAC;IACJ,CAAC;IAED,MAAM,QAAQ,GAAG,cAAc,EAAE,CAAC;IAClC,IAAI,KAAK,CAAC,MAAM,GAAG,QAAQ,EAAE,CAAC;QAC5B,OAAO;YACL,IAAI,EAAE,IAAI;YACV,MAAM,EAAE,mBAAmB,KAAK,CAAC,MAAM,MAAM,QAAQ,0BAA0B;YAC/E,OAAO;YACP,IAAI,EAAE,IAAI,CAAC,IAAI;SAChB,CAAC;IACJ,CAAC;IAED,OAAO,EAAE,IAAI,EAAE,KAAK,EAAE,OAAO,EAAE,IAAI,EAAE,IAAI,CAAC,IAAI,EAAE,CAAC;AACnD,CAAC;AAED;;;;;;;;;;;;;;;GAeG;AACH,MAAM,UAAU,iBAAiB,CAAC,IAAY;IAC5C,IAAI,IAAI,CAAC,MAAM,GAAG,MAAM;QAAE,OAAO,KAAK,CAAC;IACvC,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;IAClC,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,KAAK,CAAC;IAErC,IAAI,OAAO,GAAG,CAAC,CAAC;IAChB,KAAK,MAAM,CAAC,IAAI,KAAK;QAAE,IAAI,CAAC,CAAC,MAAM,GAAG,OAAO;YAAE,OAAO,GAAG,CAAC,CAAC,MAAM,CAAC;IAClE,IAAI,OAAO,GAAG,KAAK;QAAE,OAAO,KAAK,CAAC;IAElC,MAAM,OAAO,GAAG,IAAI,CAAC,MAAM,GAAG,KAAK,CAAC,MAAM,CAAC;IAC3C,OAAO,OAAO,GAAG,GAAG,CAAC;AACvB,CAAC"}
|
|
@@ -0,0 +1,52 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* cognium-ai#252 (problem E) — say which file is stuck.
|
|
3
|
+
*
|
|
4
|
+
* The scan logged `[N/M] Analyzing: <path>` when a worker *picked up* a file,
|
|
5
|
+
* using a counter that only advanced when a file *finished*. With four workers
|
|
6
|
+
* that produced four identical `[1/812]` lines, and when a repo wedged there
|
|
7
|
+
* was no way to tell which of the in-flight files owned the CPU — #252's
|
|
8
|
+
* evidence had to be reconstructed by attaching to the process.
|
|
9
|
+
*
|
|
10
|
+
* This tracks what is actually in flight and periodically names anything that
|
|
11
|
+
* has been running too long. It stays silent while a scan is healthy: nothing
|
|
12
|
+
* is emitted unless a file exceeds `stallMs`, so normal runs look unchanged.
|
|
13
|
+
*
|
|
14
|
+
* Env:
|
|
15
|
+
* SCAN_HEARTBEAT_MS poll interval (default 30000; 0 disables)
|
|
16
|
+
* SCAN_STALL_WARN_MS report files in flight longer than this (default 60000)
|
|
17
|
+
*/
|
|
18
|
+
export interface HeartbeatOptions {
|
|
19
|
+
/** How often to check. 0 disables the heartbeat entirely. */
|
|
20
|
+
intervalMs?: number;
|
|
21
|
+
/** Only report a file once it has been in flight this long. */
|
|
22
|
+
stallMs?: number;
|
|
23
|
+
/** Sink for the report. Defaults to stderr. */
|
|
24
|
+
emit?: (line: string) => void;
|
|
25
|
+
/** Injectable clock, for tests. */
|
|
26
|
+
now?: () => number;
|
|
27
|
+
}
|
|
28
|
+
export declare class ScanHeartbeat {
|
|
29
|
+
private inFlight;
|
|
30
|
+
private timer;
|
|
31
|
+
private readonly intervalMs;
|
|
32
|
+
private readonly stallMs;
|
|
33
|
+
private readonly emit;
|
|
34
|
+
private readonly now;
|
|
35
|
+
/** Files already named, so a long stall reports once rather than every tick. */
|
|
36
|
+
private reported;
|
|
37
|
+
constructor(opts?: HeartbeatOptions);
|
|
38
|
+
/** Record that a worker has picked up `file`. */
|
|
39
|
+
begin(file: string): void;
|
|
40
|
+
/** Record that `file` finished (or failed). */
|
|
41
|
+
end(file: string): void;
|
|
42
|
+
/**
|
|
43
|
+
* Report anything in flight past `stallMs`, longest first. Exposed for tests
|
|
44
|
+
* and callable directly; `start()` drives it on a timer.
|
|
45
|
+
*/
|
|
46
|
+
tick(): void;
|
|
47
|
+
start(): void;
|
|
48
|
+
stop(): void;
|
|
49
|
+
/** Test hook. */
|
|
50
|
+
get inFlightCount(): number;
|
|
51
|
+
}
|
|
52
|
+
//# sourceMappingURL=scan-heartbeat.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"scan-heartbeat.d.ts","sourceRoot":"","sources":["../../src/utils/scan-heartbeat.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;GAgBG;AAEH,MAAM,WAAW,gBAAgB;IAC/B,6DAA6D;IAC7D,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,+DAA+D;IAC/D,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,+CAA+C;IAC/C,IAAI,CAAC,EAAE,CAAC,IAAI,EAAE,MAAM,KAAK,IAAI,CAAC;IAC9B,mCAAmC;IACnC,GAAG,CAAC,EAAE,MAAM,MAAM,CAAC;CACpB;AAOD,qBAAa,aAAa;IACxB,OAAO,CAAC,QAAQ,CAA6B;IAC7C,OAAO,CAAC,KAAK,CAA6B;IAC1C,OAAO,CAAC,QAAQ,CAAC,UAAU,CAAS;IACpC,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAS;IACjC,OAAO,CAAC,QAAQ,CAAC,IAAI,CAAyB;IAC9C,OAAO,CAAC,QAAQ,CAAC,GAAG,CAAe;IACnC,gFAAgF;IAChF,OAAO,CAAC,QAAQ,CAAqB;gBAEzB,IAAI,GAAE,gBAAqB;IAOvC,iDAAiD;IACjD,KAAK,CAAC,IAAI,EAAE,MAAM,GAAG,IAAI;IAIzB,+CAA+C;IAC/C,GAAG,CAAC,IAAI,EAAE,MAAM,GAAG,IAAI;IAKvB;;;OAGG;IACH,IAAI,IAAI,IAAI;IAkBZ,KAAK,IAAI,IAAI;IAOb,IAAI,IAAI,IAAI;IAQZ,iBAAiB;IACjB,IAAI,aAAa,IAAI,MAAM,CAE1B;CACF"}
|
|
@@ -0,0 +1,85 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* cognium-ai#252 (problem E) — say which file is stuck.
|
|
3
|
+
*
|
|
4
|
+
* The scan logged `[N/M] Analyzing: <path>` when a worker *picked up* a file,
|
|
5
|
+
* using a counter that only advanced when a file *finished*. With four workers
|
|
6
|
+
* that produced four identical `[1/812]` lines, and when a repo wedged there
|
|
7
|
+
* was no way to tell which of the in-flight files owned the CPU — #252's
|
|
8
|
+
* evidence had to be reconstructed by attaching to the process.
|
|
9
|
+
*
|
|
10
|
+
* This tracks what is actually in flight and periodically names anything that
|
|
11
|
+
* has been running too long. It stays silent while a scan is healthy: nothing
|
|
12
|
+
* is emitted unless a file exceeds `stallMs`, so normal runs look unchanged.
|
|
13
|
+
*
|
|
14
|
+
* Env:
|
|
15
|
+
* SCAN_HEARTBEAT_MS poll interval (default 30000; 0 disables)
|
|
16
|
+
* SCAN_STALL_WARN_MS report files in flight longer than this (default 60000)
|
|
17
|
+
*/
|
|
18
|
+
function envInt(name, fallback) {
|
|
19
|
+
const raw = parseInt(process.env[name] ?? '', 10);
|
|
20
|
+
return Number.isFinite(raw) && raw >= 0 ? raw : fallback;
|
|
21
|
+
}
|
|
22
|
+
export class ScanHeartbeat {
|
|
23
|
+
inFlight = new Map();
|
|
24
|
+
timer;
|
|
25
|
+
intervalMs;
|
|
26
|
+
stallMs;
|
|
27
|
+
emit;
|
|
28
|
+
now;
|
|
29
|
+
/** Files already named, so a long stall reports once rather than every tick. */
|
|
30
|
+
reported = new Set();
|
|
31
|
+
constructor(opts = {}) {
|
|
32
|
+
this.intervalMs = opts.intervalMs ?? envInt('SCAN_HEARTBEAT_MS', 30_000);
|
|
33
|
+
this.stallMs = opts.stallMs ?? envInt('SCAN_STALL_WARN_MS', 60_000);
|
|
34
|
+
this.emit = opts.emit ?? ((l) => console.error(l));
|
|
35
|
+
this.now = opts.now ?? Date.now;
|
|
36
|
+
}
|
|
37
|
+
/** Record that a worker has picked up `file`. */
|
|
38
|
+
begin(file) {
|
|
39
|
+
this.inFlight.set(file, this.now());
|
|
40
|
+
}
|
|
41
|
+
/** Record that `file` finished (or failed). */
|
|
42
|
+
end(file) {
|
|
43
|
+
this.inFlight.delete(file);
|
|
44
|
+
this.reported.delete(file);
|
|
45
|
+
}
|
|
46
|
+
/**
|
|
47
|
+
* Report anything in flight past `stallMs`, longest first. Exposed for tests
|
|
48
|
+
* and callable directly; `start()` drives it on a timer.
|
|
49
|
+
*/
|
|
50
|
+
tick() {
|
|
51
|
+
const t = this.now();
|
|
52
|
+
const stalled = [...this.inFlight.entries()]
|
|
53
|
+
.filter(([file, since]) => t - since >= this.stallMs && !this.reported.has(file))
|
|
54
|
+
.sort((a, b) => a[1] - b[1]);
|
|
55
|
+
for (const [file, since] of stalled) {
|
|
56
|
+
this.reported.add(file);
|
|
57
|
+
const elapsedMs = t - since;
|
|
58
|
+
// Sub-second thresholds are only used in tests, but "after 0s" reads
|
|
59
|
+
// like a bug, so keep the unit honest.
|
|
60
|
+
const elapsed = elapsedMs < 1000 ? `${elapsedMs}ms` : `${Math.round(elapsedMs / 1000)}s`;
|
|
61
|
+
this.emit(` ⏱ still analyzing after ${elapsed}: ${file} ` +
|
|
62
|
+
`(${this.inFlight.size} file(s) in flight). cognium-ai#252`);
|
|
63
|
+
}
|
|
64
|
+
}
|
|
65
|
+
start() {
|
|
66
|
+
if (this.intervalMs <= 0 || this.timer)
|
|
67
|
+
return;
|
|
68
|
+
this.timer = setInterval(() => this.tick(), this.intervalMs);
|
|
69
|
+
// Never hold the process open on account of the heartbeat.
|
|
70
|
+
this.timer.unref?.();
|
|
71
|
+
}
|
|
72
|
+
stop() {
|
|
73
|
+
if (this.timer)
|
|
74
|
+
clearTimeout(this.timer);
|
|
75
|
+
clearInterval(this.timer);
|
|
76
|
+
this.timer = undefined;
|
|
77
|
+
this.inFlight.clear();
|
|
78
|
+
this.reported.clear();
|
|
79
|
+
}
|
|
80
|
+
/** Test hook. */
|
|
81
|
+
get inFlightCount() {
|
|
82
|
+
return this.inFlight.size;
|
|
83
|
+
}
|
|
84
|
+
}
|
|
85
|
+
//# sourceMappingURL=scan-heartbeat.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"scan-heartbeat.js","sourceRoot":"","sources":["../../src/utils/scan-heartbeat.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;GAgBG;AAaH,SAAS,MAAM,CAAC,IAAY,EAAE,QAAgB;IAC5C,MAAM,GAAG,GAAG,QAAQ,CAAC,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,EAAE,EAAE,EAAE,CAAC,CAAC;IAClD,OAAO,MAAM,CAAC,QAAQ,CAAC,GAAG,CAAC,IAAI,GAAG,IAAI,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,QAAQ,CAAC;AAC3D,CAAC;AAED,MAAM,OAAO,aAAa;IAChB,QAAQ,GAAG,IAAI,GAAG,EAAkB,CAAC;IACrC,KAAK,CAA6B;IACzB,UAAU,CAAS;IACnB,OAAO,CAAS;IAChB,IAAI,CAAyB;IAC7B,GAAG,CAAe;IACnC,gFAAgF;IACxE,QAAQ,GAAG,IAAI,GAAG,EAAU,CAAC;IAErC,YAAY,OAAyB,EAAE;QACrC,IAAI,CAAC,UAAU,GAAG,IAAI,CAAC,UAAU,IAAI,MAAM,CAAC,mBAAmB,EAAE,MAAM,CAAC,CAAC;QACzE,IAAI,CAAC,OAAO,GAAG,IAAI,CAAC,OAAO,IAAI,MAAM,CAAC,oBAAoB,EAAE,MAAM,CAAC,CAAC;QACpE,IAAI,CAAC,IAAI,GAAG,IAAI,CAAC,IAAI,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC;QACnD,IAAI,CAAC,GAAG,GAAG,IAAI,CAAC,GAAG,IAAI,IAAI,CAAC,GAAG,CAAC;IAClC,CAAC;IAED,iDAAiD;IACjD,KAAK,CAAC,IAAY;QAChB,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,IAAI,EAAE,IAAI,CAAC,GAAG,EAAE,CAAC,CAAC;IACtC,CAAC;IAED,+CAA+C;IAC/C,GAAG,CAAC,IAAY;QACd,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;QAC3B,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;IAC7B,CAAC;IAED;;;OAGG;IACH,IAAI;QACF,MAAM,CAAC,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QACrB,MAAM,OAAO,GAAG,CAAC,GAAG,IAAI,CAAC,QAAQ,CAAC,OAAO,EAAE,CAAC;aACzC,MAAM,CAAC,CAAC,CAAC,IAAI,EAAE,KAAK,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,KAAK,IAAI,IAAI,CAAC,OAAO,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;aAChF,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;QAC/B,KAAK,MAAM,CAAC,IAAI,EAAE,KAAK,CAAC,IAAI,OAAO,EAAE,CAAC;YACpC,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;YACxB,MAAM,SAAS,GAAG,CAAC,GAAG,KAAK,CAAC;YAC5B,qEAAqE;YACrE,uCAAuC;YACvC,MAAM,OAAO,GAAG,SAAS,GAAG,IAAI,CAAC,CAAC,CAAC,GAAG,SAAS,IAAI,CAAC,CAAC,CAAC,GAAG,IAAI,CAAC,KAAK,CAAC,SAAS,GAAG,IAAI,CAAC,GAAG,CAAC;YACzF,IAAI,CAAC,IAAI,CACP,6BAA6B,OAAO,KAAK,IAAI,GAAG;gBAC9C,IAAI,IAAI,CAAC,QAAQ,CAAC,IAAI,qCAAqC,CAC9D,CAAC;QACJ,CAAC;IACH,CAAC;IAED,KAAK;QACH,IAAI,IAAI,CAAC,UAAU,IAAI,CAAC,IAAI,IAAI,CAAC,KAAK;YAAE,OAAO;QAC/C,IAAI,CAAC,KAAK,GAAG,WAAW,CAAC,GAAG,EAAE,CAAC,IAAI,CAAC,IAAI,EAAE,EAAE,IAAI,CAAC,UAAU,CAAC,CAAC;QAC7D,2DAA2D;QAC3D,IAAI,CAAC,KAAK,CAAC,KAAK,EAAE,EAAE,CAAC;IACvB,CAAC;IAED,IAAI;QACF,IAAI,IAAI,CAAC,KAAK;YAAE,YAAY,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;QACzC,aAAa,CAAC,IAAI,CAAC,KAAuB,CAAC,CAAC;QAC5C,IAAI,CAAC,KAAK,GAAG,SAAS,CAAC;QACvB,IAAI,CAAC,QAAQ,CAAC,KAAK,EAAE,CAAC;QACtB,IAAI,CAAC,QAAQ,CAAC,KAAK,EAAE,CAAC;IACxB,CAAC;IAED,iBAAiB;IACjB,IAAI,aAAa;QACf,OAAO,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC;IAC5B,CAAC;CACF"}
|