@reportforge/playwright-pdf 0.19.0 → 0.20.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +31 -0
- package/README.md +3 -2
- package/dist/cli/index.js +1017 -183
- package/dist/index.d.mts +26 -2
- package/dist/index.d.ts +26 -2
- package/dist/index.js +311 -156
- package/dist/index.mjs +386 -231
- package/package.json +1 -1
package/dist/cli/index.js
CHANGED
|
@@ -1588,13 +1588,13 @@ var require_ast = __commonJS({
|
|
|
1588
1588
|
helperExpression: function helperExpression(node) {
|
|
1589
1589
|
return node.type === "SubExpression" || (node.type === "MustacheStatement" || node.type === "BlockStatement") && !!(node.params && node.params.length || node.hash);
|
|
1590
1590
|
},
|
|
1591
|
-
scopedId: function scopedId(
|
|
1592
|
-
return /^\.|this\b/.test(
|
|
1591
|
+
scopedId: function scopedId(path14) {
|
|
1592
|
+
return /^\.|this\b/.test(path14.original);
|
|
1593
1593
|
},
|
|
1594
1594
|
// an ID is simple if it only has one part, and that part is not
|
|
1595
1595
|
// `..` or `this`.
|
|
1596
|
-
simpleId: function simpleId(
|
|
1597
|
-
return
|
|
1596
|
+
simpleId: function simpleId(path14) {
|
|
1597
|
+
return path14.parts.length === 1 && !AST.helpers.scopedId(path14) && !path14.depth;
|
|
1598
1598
|
}
|
|
1599
1599
|
}
|
|
1600
1600
|
};
|
|
@@ -2668,12 +2668,12 @@ var require_helpers2 = __commonJS({
|
|
|
2668
2668
|
loc
|
|
2669
2669
|
};
|
|
2670
2670
|
}
|
|
2671
|
-
function prepareMustache(
|
|
2671
|
+
function prepareMustache(path14, params, hash, open, strip, locInfo) {
|
|
2672
2672
|
var escapeFlag = open.charAt(3) || open.charAt(2), escaped = escapeFlag !== "{" && escapeFlag !== "&";
|
|
2673
2673
|
var decorator = /\*/.test(open);
|
|
2674
2674
|
return {
|
|
2675
2675
|
type: decorator ? "Decorator" : "MustacheStatement",
|
|
2676
|
-
path:
|
|
2676
|
+
path: path14,
|
|
2677
2677
|
params,
|
|
2678
2678
|
hash,
|
|
2679
2679
|
escaped,
|
|
@@ -2993,9 +2993,9 @@ var require_compiler = __commonJS({
|
|
|
2993
2993
|
},
|
|
2994
2994
|
DecoratorBlock: function DecoratorBlock(decorator) {
|
|
2995
2995
|
var program = decorator.program && this.compileProgram(decorator.program);
|
|
2996
|
-
var params = this.setupFullMustacheParams(decorator, program, void 0),
|
|
2996
|
+
var params = this.setupFullMustacheParams(decorator, program, void 0), path14 = decorator.path;
|
|
2997
2997
|
this.useDecorators = true;
|
|
2998
|
-
this.opcode("registerDecorator", params.length,
|
|
2998
|
+
this.opcode("registerDecorator", params.length, path14.original);
|
|
2999
2999
|
},
|
|
3000
3000
|
PartialStatement: function PartialStatement(partial) {
|
|
3001
3001
|
this.usePartial = true;
|
|
@@ -3059,46 +3059,46 @@ var require_compiler = __commonJS({
|
|
|
3059
3059
|
}
|
|
3060
3060
|
},
|
|
3061
3061
|
ambiguousSexpr: function ambiguousSexpr(sexpr, program, inverse) {
|
|
3062
|
-
var
|
|
3063
|
-
this.opcode("getContext",
|
|
3062
|
+
var path14 = sexpr.path, name = path14.parts[0], isBlock = program != null || inverse != null;
|
|
3063
|
+
this.opcode("getContext", path14.depth);
|
|
3064
3064
|
this.opcode("pushProgram", program);
|
|
3065
3065
|
this.opcode("pushProgram", inverse);
|
|
3066
|
-
|
|
3067
|
-
this.accept(
|
|
3066
|
+
path14.strict = true;
|
|
3067
|
+
this.accept(path14);
|
|
3068
3068
|
this.opcode("invokeAmbiguous", name, isBlock);
|
|
3069
3069
|
},
|
|
3070
3070
|
simpleSexpr: function simpleSexpr(sexpr) {
|
|
3071
|
-
var
|
|
3072
|
-
|
|
3073
|
-
this.accept(
|
|
3071
|
+
var path14 = sexpr.path;
|
|
3072
|
+
path14.strict = true;
|
|
3073
|
+
this.accept(path14);
|
|
3074
3074
|
this.opcode("resolvePossibleLambda");
|
|
3075
3075
|
},
|
|
3076
3076
|
helperSexpr: function helperSexpr(sexpr, program, inverse) {
|
|
3077
|
-
var params = this.setupFullMustacheParams(sexpr, program, inverse),
|
|
3077
|
+
var params = this.setupFullMustacheParams(sexpr, program, inverse), path14 = sexpr.path, name = path14.parts[0];
|
|
3078
3078
|
if (this.options.knownHelpers[name]) {
|
|
3079
3079
|
this.opcode("invokeKnownHelper", params.length, name);
|
|
3080
3080
|
} else if (this.options.knownHelpersOnly) {
|
|
3081
3081
|
throw new _exception2["default"]("You specified knownHelpersOnly, but used the unknown helper " + name, sexpr);
|
|
3082
3082
|
} else {
|
|
3083
|
-
|
|
3084
|
-
|
|
3085
|
-
this.accept(
|
|
3086
|
-
this.opcode("invokeHelper", params.length,
|
|
3083
|
+
path14.strict = true;
|
|
3084
|
+
path14.falsy = true;
|
|
3085
|
+
this.accept(path14);
|
|
3086
|
+
this.opcode("invokeHelper", params.length, path14.original, _ast2["default"].helpers.simpleId(path14));
|
|
3087
3087
|
}
|
|
3088
3088
|
},
|
|
3089
|
-
PathExpression: function PathExpression(
|
|
3090
|
-
this.addDepth(
|
|
3091
|
-
this.opcode("getContext",
|
|
3092
|
-
var name =
|
|
3089
|
+
PathExpression: function PathExpression(path14) {
|
|
3090
|
+
this.addDepth(path14.depth);
|
|
3091
|
+
this.opcode("getContext", path14.depth);
|
|
3092
|
+
var name = path14.parts[0], scoped = _ast2["default"].helpers.scopedId(path14), blockParamId = !path14.depth && !scoped && this.blockParamIndex(name);
|
|
3093
3093
|
if (blockParamId) {
|
|
3094
|
-
this.opcode("lookupBlockParam", blockParamId,
|
|
3094
|
+
this.opcode("lookupBlockParam", blockParamId, path14.parts);
|
|
3095
3095
|
} else if (!name) {
|
|
3096
3096
|
this.opcode("pushContext");
|
|
3097
|
-
} else if (
|
|
3097
|
+
} else if (path14.data) {
|
|
3098
3098
|
this.options.data = true;
|
|
3099
|
-
this.opcode("lookupData",
|
|
3099
|
+
this.opcode("lookupData", path14.depth, path14.parts, path14.strict);
|
|
3100
3100
|
} else {
|
|
3101
|
-
this.opcode("lookupOnContext",
|
|
3101
|
+
this.opcode("lookupOnContext", path14.parts, path14.falsy, path14.strict, scoped);
|
|
3102
3102
|
}
|
|
3103
3103
|
},
|
|
3104
3104
|
StringLiteral: function StringLiteral(string) {
|
|
@@ -3454,16 +3454,16 @@ var require_util = __commonJS({
|
|
|
3454
3454
|
}
|
|
3455
3455
|
exports2.urlGenerate = urlGenerate;
|
|
3456
3456
|
function normalize(aPath) {
|
|
3457
|
-
var
|
|
3457
|
+
var path14 = aPath;
|
|
3458
3458
|
var url = urlParse(aPath);
|
|
3459
3459
|
if (url) {
|
|
3460
3460
|
if (!url.path) {
|
|
3461
3461
|
return aPath;
|
|
3462
3462
|
}
|
|
3463
|
-
|
|
3463
|
+
path14 = url.path;
|
|
3464
3464
|
}
|
|
3465
|
-
var isAbsolute = exports2.isAbsolute(
|
|
3466
|
-
var parts =
|
|
3465
|
+
var isAbsolute = exports2.isAbsolute(path14);
|
|
3466
|
+
var parts = path14.split(/\/+/);
|
|
3467
3467
|
for (var part, up = 0, i = parts.length - 1; i >= 0; i--) {
|
|
3468
3468
|
part = parts[i];
|
|
3469
3469
|
if (part === ".") {
|
|
@@ -3480,15 +3480,15 @@ var require_util = __commonJS({
|
|
|
3480
3480
|
}
|
|
3481
3481
|
}
|
|
3482
3482
|
}
|
|
3483
|
-
|
|
3484
|
-
if (
|
|
3485
|
-
|
|
3483
|
+
path14 = parts.join("/");
|
|
3484
|
+
if (path14 === "") {
|
|
3485
|
+
path14 = isAbsolute ? "/" : ".";
|
|
3486
3486
|
}
|
|
3487
3487
|
if (url) {
|
|
3488
|
-
url.path =
|
|
3488
|
+
url.path = path14;
|
|
3489
3489
|
return urlGenerate(url);
|
|
3490
3490
|
}
|
|
3491
|
-
return
|
|
3491
|
+
return path14;
|
|
3492
3492
|
}
|
|
3493
3493
|
exports2.normalize = normalize;
|
|
3494
3494
|
function join(aRoot, aPath) {
|
|
@@ -6291,8 +6291,8 @@ var require_printer = __commonJS({
|
|
|
6291
6291
|
return this.accept(sexpr.path) + " " + params + hash;
|
|
6292
6292
|
};
|
|
6293
6293
|
PrintVisitor.prototype.PathExpression = function(id) {
|
|
6294
|
-
var
|
|
6295
|
-
return (id.data ? "@" : "") + "PATH:" +
|
|
6294
|
+
var path14 = id.parts.join("/");
|
|
6295
|
+
return (id.data ? "@" : "") + "PATH:" + path14;
|
|
6296
6296
|
};
|
|
6297
6297
|
PrintVisitor.prototype.StringLiteral = function(string) {
|
|
6298
6298
|
return '"' + string.value + '"';
|
|
@@ -6333,8 +6333,8 @@ var require_lib = __commonJS({
|
|
|
6333
6333
|
handlebars.print = printer.print;
|
|
6334
6334
|
module2.exports = handlebars;
|
|
6335
6335
|
function extension(module3, filename) {
|
|
6336
|
-
var
|
|
6337
|
-
var templateString =
|
|
6336
|
+
var fs14 = require("fs");
|
|
6337
|
+
var templateString = fs14.readFileSync(filename, "utf8");
|
|
6338
6338
|
module3.exports = handlebars.compile(templateString);
|
|
6339
6339
|
}
|
|
6340
6340
|
if (typeof require !== "undefined" && require.extensions) {
|
|
@@ -6854,8 +6854,8 @@ var init_engine = __esm({
|
|
|
6854
6854
|
};
|
|
6855
6855
|
return map[v] ?? (typeof v === "string" ? v.toUpperCase() : v);
|
|
6856
6856
|
});
|
|
6857
|
-
this.handlebars.registerHelper("coverageBand", (
|
|
6858
|
-
const n = Number(
|
|
6857
|
+
this.handlebars.registerHelper("coverageBand", (pct2) => {
|
|
6858
|
+
const n = Number(pct2);
|
|
6859
6859
|
return n >= 80 ? "high" : n >= 50 ? "mid" : "low";
|
|
6860
6860
|
});
|
|
6861
6861
|
this.handlebars.registerHelper("addOne", (n) => (n ?? 0) + 1);
|
|
@@ -7005,7 +7005,7 @@ var require_util2 = __commonJS({
|
|
|
7005
7005
|
"node_modules/chrome-finder/lib/util.js"(exports2, module2) {
|
|
7006
7006
|
"use strict";
|
|
7007
7007
|
init_cjs_shims();
|
|
7008
|
-
var
|
|
7008
|
+
var fs14 = require("fs");
|
|
7009
7009
|
var newLineRegex = /\r?\n/;
|
|
7010
7010
|
function sort(installations, priorities) {
|
|
7011
7011
|
const defaultPriority = 10;
|
|
@@ -7023,7 +7023,7 @@ var require_util2 = __commonJS({
|
|
|
7023
7023
|
return false;
|
|
7024
7024
|
}
|
|
7025
7025
|
try {
|
|
7026
|
-
|
|
7026
|
+
fs14.accessSync(file);
|
|
7027
7027
|
return true;
|
|
7028
7028
|
} catch (e) {
|
|
7029
7029
|
return false;
|
|
@@ -7034,7 +7034,7 @@ var require_util2 = __commonJS({
|
|
|
7034
7034
|
return false;
|
|
7035
7035
|
}
|
|
7036
7036
|
try {
|
|
7037
|
-
var stat =
|
|
7037
|
+
var stat = fs14.statSync(file);
|
|
7038
7038
|
return stat && typeof stat.isFile === "function" && stat.isFile();
|
|
7039
7039
|
} catch (e) {
|
|
7040
7040
|
return false;
|
|
@@ -7054,7 +7054,7 @@ var require_win32 = __commonJS({
|
|
|
7054
7054
|
"node_modules/chrome-finder/lib/win32.js"(exports2, module2) {
|
|
7055
7055
|
"use strict";
|
|
7056
7056
|
init_cjs_shims();
|
|
7057
|
-
var
|
|
7057
|
+
var path14 = require("path");
|
|
7058
7058
|
var { canAccess } = require_util2();
|
|
7059
7059
|
function win32() {
|
|
7060
7060
|
const installations = [];
|
|
@@ -7068,7 +7068,7 @@ var require_win32 = __commonJS({
|
|
|
7068
7068
|
const prefixes = [process.env.LOCALAPPDATA, process.env.PROGRAMFILES, process.env["PROGRAMFILES(X86)"]];
|
|
7069
7069
|
prefixes.forEach((prefix) => suffixes.forEach((suffix) => {
|
|
7070
7070
|
if (prefix) {
|
|
7071
|
-
const chromePath =
|
|
7071
|
+
const chromePath = path14.join(prefix, suffix);
|
|
7072
7072
|
if (canAccess(chromePath)) {
|
|
7073
7073
|
installations.push(chromePath);
|
|
7074
7074
|
}
|
|
@@ -7086,7 +7086,7 @@ var require_darwin = __commonJS({
|
|
|
7086
7086
|
"use strict";
|
|
7087
7087
|
init_cjs_shims();
|
|
7088
7088
|
var { execSync: execSync2 } = require("child_process");
|
|
7089
|
-
var
|
|
7089
|
+
var path14 = require("path");
|
|
7090
7090
|
var { canAccess, newLineRegex, sort } = require_util2();
|
|
7091
7091
|
function darwin() {
|
|
7092
7092
|
const suffixes = ["/Contents/MacOS/Google Chrome Canary", "/Contents/MacOS/Google Chrome", "/Contents/MacOS/Chromium"];
|
|
@@ -7096,7 +7096,7 @@ var require_darwin = __commonJS({
|
|
|
7096
7096
|
${LSREGISTER} -dump | grep -E -i -o '/.+(google chrome( canary)?|chromium)\\.app(\\s|$)' | grep -E -v 'Caches|TimeMachine|Temporary|/Volumes|\\.Trash'
|
|
7097
7097
|
`).toString().split(newLineRegex).forEach((inst) => {
|
|
7098
7098
|
suffixes.forEach((suffix) => {
|
|
7099
|
-
const execPath =
|
|
7099
|
+
const execPath = path14.join(inst.trim(), suffix);
|
|
7100
7100
|
if (canAccess(execPath)) {
|
|
7101
7101
|
installations.push(execPath);
|
|
7102
7102
|
}
|
|
@@ -7125,8 +7125,8 @@ var require_linux = __commonJS({
|
|
|
7125
7125
|
"use strict";
|
|
7126
7126
|
init_cjs_shims();
|
|
7127
7127
|
var { execSync: execSync2, execFileSync: execFileSync2 } = require("child_process");
|
|
7128
|
-
var
|
|
7129
|
-
var
|
|
7128
|
+
var path14 = require("path");
|
|
7129
|
+
var fs14 = require("fs");
|
|
7130
7130
|
var { canAccess, sort, isExecutable, newLineRegex } = require_util2();
|
|
7131
7131
|
function findChromeExecutablesForLinuxDesktop(folder) {
|
|
7132
7132
|
const argumentsRegex = /(^[^ ]+).*/;
|
|
@@ -7143,7 +7143,7 @@ var require_linux = __commonJS({
|
|
|
7143
7143
|
function linux() {
|
|
7144
7144
|
let installations = [];
|
|
7145
7145
|
const desktopInstallationFolders = [
|
|
7146
|
-
|
|
7146
|
+
path14.join(require("os").homedir(), ".local/share/applications/"),
|
|
7147
7147
|
"/usr/share/applications/"
|
|
7148
7148
|
];
|
|
7149
7149
|
desktopInstallationFolders.forEach((folder) => {
|
|
@@ -7169,7 +7169,7 @@ var require_linux = __commonJS({
|
|
|
7169
7169
|
].map((possiblePath) => {
|
|
7170
7170
|
try {
|
|
7171
7171
|
const chromePathToTest = possiblePath + "/" + executable;
|
|
7172
|
-
if (
|
|
7172
|
+
if (fs14.existsSync(chromePathToTest) && canAccess(chromePathToTest) && isExecutable(chromePathToTest)) {
|
|
7173
7173
|
installations.push(chromePathToTest);
|
|
7174
7174
|
return chromePathToTest;
|
|
7175
7175
|
}
|
|
@@ -7236,7 +7236,7 @@ var require_lib2 = __commonJS({
|
|
|
7236
7236
|
});
|
|
7237
7237
|
|
|
7238
7238
|
// src/pdf/BrowserManager.ts
|
|
7239
|
-
var import_child_process, import_fs3, BrowserManager;
|
|
7239
|
+
var import_child_process, import_fs3, LAUNCH_ARGS, BrowserManager;
|
|
7240
7240
|
var init_BrowserManager = __esm({
|
|
7241
7241
|
"src/pdf/BrowserManager.ts"() {
|
|
7242
7242
|
"use strict";
|
|
@@ -7244,21 +7244,26 @@ var init_BrowserManager = __esm({
|
|
|
7244
7244
|
import_child_process = require("child_process");
|
|
7245
7245
|
import_fs3 = __toESM(require("fs"));
|
|
7246
7246
|
init_logger();
|
|
7247
|
+
LAUNCH_ARGS = [
|
|
7248
|
+
"--no-sandbox",
|
|
7249
|
+
"--disable-setuid-sandbox",
|
|
7250
|
+
"--disable-dev-shm-usage",
|
|
7251
|
+
"--disable-gpu",
|
|
7252
|
+
// Chrome's new headless mode (--headless=new, the only mode left in
|
|
7253
|
+
// Chrome 132+) briefly uncloaks a blank window on some Windows/GPU
|
|
7254
|
+
// combinations. Parking the window far off-screen makes that transient
|
|
7255
|
+
// invisible; harmless everywhere else and does not affect PDF output.
|
|
7256
|
+
"--window-position=-32000,-32000"
|
|
7257
|
+
];
|
|
7247
7258
|
BrowserManager = class {
|
|
7248
7259
|
async launch(executablePath) {
|
|
7249
7260
|
const puppeteer = require("puppeteer-core");
|
|
7250
7261
|
const chromePath = this.resolveChromePath(executablePath, puppeteer);
|
|
7251
7262
|
logger.info(`Launching browser: ${chromePath ?? "puppeteer default"}`);
|
|
7252
|
-
const launchArgs = [
|
|
7253
|
-
"--no-sandbox",
|
|
7254
|
-
"--disable-setuid-sandbox",
|
|
7255
|
-
"--disable-dev-shm-usage",
|
|
7256
|
-
"--disable-gpu"
|
|
7257
|
-
];
|
|
7258
7263
|
this.browser = await puppeteer.launch({
|
|
7259
7264
|
executablePath: chromePath,
|
|
7260
7265
|
headless: true,
|
|
7261
|
-
args:
|
|
7266
|
+
args: LAUNCH_ARGS
|
|
7262
7267
|
});
|
|
7263
7268
|
const page = await this.browser.newPage();
|
|
7264
7269
|
return { browser: this.browser, page };
|
|
@@ -8297,8 +8302,8 @@ var init_parseUtil = __esm({
|
|
|
8297
8302
|
init_errors();
|
|
8298
8303
|
init_en();
|
|
8299
8304
|
makeIssue = (params) => {
|
|
8300
|
-
const { data, path:
|
|
8301
|
-
const fullPath = [...
|
|
8305
|
+
const { data, path: path14, errorMaps, issueData } = params;
|
|
8306
|
+
const fullPath = [...path14, ...issueData.path || []];
|
|
8302
8307
|
const fullIssue = {
|
|
8303
8308
|
...issueData,
|
|
8304
8309
|
path: fullPath
|
|
@@ -8612,11 +8617,11 @@ var init_types = __esm({
|
|
|
8612
8617
|
init_parseUtil();
|
|
8613
8618
|
init_util();
|
|
8614
8619
|
ParseInputLazyPath = class {
|
|
8615
|
-
constructor(parent, value,
|
|
8620
|
+
constructor(parent, value, path14, key) {
|
|
8616
8621
|
this._cachedPath = [];
|
|
8617
8622
|
this.parent = parent;
|
|
8618
8623
|
this.data = value;
|
|
8619
|
-
this._path =
|
|
8624
|
+
this._path = path14;
|
|
8620
8625
|
this._key = key;
|
|
8621
8626
|
}
|
|
8622
8627
|
get path() {
|
|
@@ -12049,9 +12054,17 @@ var init_schema = __esm({
|
|
|
12049
12054
|
// Qualitative floor (not a numeric % — consistent with the match-strength design).
|
|
12050
12055
|
minStrength: external_exports.enum(["weak", "moderate", "strong"]).default("weak"),
|
|
12051
12056
|
maxFailuresToAnalyse: external_exports.number().int().min(1).default(500),
|
|
12052
|
-
// Consumed by the Plan 3 feedback collector; inert until then.
|
|
12053
12057
|
collectUnclassified: external_exports.boolean().default(true),
|
|
12054
|
-
|
|
12058
|
+
// What the feedback collector stores (while collectUnclassified is on):
|
|
12059
|
+
// 'blind-spots' = unknown/low-confidence only; 'all' = every model-scored
|
|
12060
|
+
// failure, so wrong-but-confident verdicts can be corrected by labeling.
|
|
12061
|
+
collectScope: external_exports.enum(["blind-spots", "all"]).default("blind-spots"),
|
|
12062
|
+
autoUpdateModel: external_exports.boolean().default(true),
|
|
12063
|
+
// Use the on-device trained model (when present and its evaluation gate passed).
|
|
12064
|
+
localModel: external_exports.boolean().default(true),
|
|
12065
|
+
// Override the local model file location (default ~/.reportforge/model-local.json).
|
|
12066
|
+
// Point at a repo-committed file to share a team-trained model with ephemeral CI.
|
|
12067
|
+
localModelPath: external_exports.string().optional()
|
|
12055
12068
|
}).default({});
|
|
12056
12069
|
captureConfigSchema = external_exports.object({
|
|
12057
12070
|
steps: external_exports.boolean().default(false),
|
|
@@ -12152,6 +12165,56 @@ var init_model_bundle = __esm({
|
|
|
12152
12165
|
}
|
|
12153
12166
|
});
|
|
12154
12167
|
|
|
12168
|
+
// src/analysis/tokenize.ts
|
|
12169
|
+
function tokenize(text) {
|
|
12170
|
+
return text.toLowerCase().replace(/https?:\/\/\S+/gi, " URL ").replace(/[^a-zA-Z0-9\s]/g, " ").replace(/\b\d{3,}[a-z]*\b/g, " NUM ").split(/\s+/).filter((t) => t.length > 1);
|
|
12171
|
+
}
|
|
12172
|
+
var init_tokenize = __esm({
|
|
12173
|
+
"src/analysis/tokenize.ts"() {
|
|
12174
|
+
"use strict";
|
|
12175
|
+
init_cjs_shims();
|
|
12176
|
+
}
|
|
12177
|
+
});
|
|
12178
|
+
|
|
12179
|
+
// src/analysis/NaiveBayesClassifier.ts
|
|
12180
|
+
function classify(model, text) {
|
|
12181
|
+
return classifyTokens(model, tokenize(text));
|
|
12182
|
+
}
|
|
12183
|
+
function classifyTokens(model, tokens) {
|
|
12184
|
+
const vocabTokens = tokens.filter(
|
|
12185
|
+
(tok) => tok.length > 0 && model.labels.some((l) => model.logLikelihoods[l]?.[tok] !== void 0)
|
|
12186
|
+
);
|
|
12187
|
+
if (vocabTokens.length === 0) {
|
|
12188
|
+
return { label: "unknown", margin: 0 };
|
|
12189
|
+
}
|
|
12190
|
+
const scores = {};
|
|
12191
|
+
for (const label of model.labels) {
|
|
12192
|
+
let s = model.priors[label];
|
|
12193
|
+
const oov = Math.log(model.alpha / (model.totals[label] + model.alpha * model.vocabSize));
|
|
12194
|
+
const ll = model.logLikelihoods[label];
|
|
12195
|
+
for (const tok of vocabTokens) {
|
|
12196
|
+
s += ll[tok] ?? oov;
|
|
12197
|
+
}
|
|
12198
|
+
scores[label] = s;
|
|
12199
|
+
}
|
|
12200
|
+
const ranked = Object.entries(scores).sort((a, b) => b[1] - a[1]);
|
|
12201
|
+
const topLog = ranked[0][1];
|
|
12202
|
+
const secondLog = ranked[1]?.[1] ?? topLog;
|
|
12203
|
+
const margin = (topLog - secondLog) / vocabTokens.length;
|
|
12204
|
+
return { label: ranked[0][0], margin };
|
|
12205
|
+
}
|
|
12206
|
+
function isValidNaiveBayesModel(m) {
|
|
12207
|
+
const x = m;
|
|
12208
|
+
return !!x && Array.isArray(x.labels) && x.labels.length > 0 && typeof x.version === "number" && !!x.priors && !!x.totals && !!x.logLikelihoods && typeof x.vocabSize === "number" && typeof x.alpha === "number";
|
|
12209
|
+
}
|
|
12210
|
+
var init_NaiveBayesClassifier = __esm({
|
|
12211
|
+
"src/analysis/NaiveBayesClassifier.ts"() {
|
|
12212
|
+
"use strict";
|
|
12213
|
+
init_cjs_shims();
|
|
12214
|
+
init_tokenize();
|
|
12215
|
+
}
|
|
12216
|
+
});
|
|
12217
|
+
|
|
12155
12218
|
// src/analysis/ModelStore.ts
|
|
12156
12219
|
function cacheFile() {
|
|
12157
12220
|
return testDeps?.cacheFile ?? import_node_path.default.join(import_node_os.default.homedir(), ".reportforge", "model.json");
|
|
@@ -12173,10 +12236,6 @@ function verifyModelSignature(modelJson, signatureB64url) {
|
|
|
12173
12236
|
return false;
|
|
12174
12237
|
}
|
|
12175
12238
|
}
|
|
12176
|
-
function schemaValid(m) {
|
|
12177
|
-
const x = m;
|
|
12178
|
-
return !!x && Array.isArray(x.labels) && x.labels.length > 0 && typeof x.version === "number" && !!x.priors && !!x.totals && !!x.logLikelihoods && typeof x.vocabSize === "number" && typeof x.alpha === "number";
|
|
12179
|
-
}
|
|
12180
12239
|
function readCache() {
|
|
12181
12240
|
try {
|
|
12182
12241
|
const raw = import_node_fs.default.readFileSync(cacheFile(), "utf8");
|
|
@@ -12205,7 +12264,7 @@ function loadModel(autoUpdate) {
|
|
|
12205
12264
|
}
|
|
12206
12265
|
return MODEL;
|
|
12207
12266
|
}
|
|
12208
|
-
var import_node_fs, import_node_os, import_node_path, import_node_crypto, MODEL_SIG_DOMAIN, testDeps;
|
|
12267
|
+
var import_node_fs, import_node_os, import_node_path, import_node_crypto, MODEL_SIG_DOMAIN, testDeps, schemaValid;
|
|
12209
12268
|
var init_ModelStore = __esm({
|
|
12210
12269
|
"src/analysis/ModelStore.ts"() {
|
|
12211
12270
|
"use strict";
|
|
@@ -12216,62 +12275,114 @@ var init_ModelStore = __esm({
|
|
|
12216
12275
|
import_node_crypto = require("crypto");
|
|
12217
12276
|
init_pubkey_bundle();
|
|
12218
12277
|
init_model_bundle();
|
|
12278
|
+
init_NaiveBayesClassifier();
|
|
12219
12279
|
init_logger();
|
|
12220
12280
|
MODEL_SIG_DOMAIN = "reportforge-model-sig:v1\n";
|
|
12221
12281
|
testDeps = null;
|
|
12282
|
+
schemaValid = isValidNaiveBayesModel;
|
|
12222
12283
|
}
|
|
12223
12284
|
});
|
|
12224
12285
|
|
|
12225
|
-
// src/analysis/
|
|
12226
|
-
function
|
|
12227
|
-
return
|
|
12286
|
+
// src/analysis/LocalModelStore.ts
|
|
12287
|
+
function defaultLocalModelPath() {
|
|
12288
|
+
return testDeps2?.file ?? import_node_path2.default.join(import_node_os2.default.homedir(), ".reportforge", "model-local.json");
|
|
12228
12289
|
}
|
|
12229
|
-
|
|
12230
|
-
|
|
12290
|
+
function loadLocalModel(overridePath) {
|
|
12291
|
+
const file = overridePath ?? defaultLocalModelPath();
|
|
12292
|
+
const explicit = overridePath !== void 0;
|
|
12293
|
+
const reject = (why) => {
|
|
12294
|
+
const msg = `local model at ${file} not used: ${why}`;
|
|
12295
|
+
if (explicit) logger.warn(msg);
|
|
12296
|
+
else logger.debug(msg);
|
|
12297
|
+
return null;
|
|
12298
|
+
};
|
|
12299
|
+
let parsed;
|
|
12300
|
+
try {
|
|
12301
|
+
parsed = JSON.parse(import_node_fs2.default.readFileSync(file, "utf8"));
|
|
12302
|
+
} catch (e) {
|
|
12303
|
+
return explicit ? reject(`unreadable (${e.message})`) : null;
|
|
12304
|
+
}
|
|
12305
|
+
if (parsed.formatVersion !== 1) return reject(`unsupported formatVersion ${String(parsed.formatVersion)}`);
|
|
12306
|
+
if (!isValidNaiveBayesModel(parsed.model)) return reject("embedded model failed schema validation");
|
|
12307
|
+
const meta = parsed.meta;
|
|
12308
|
+
if (!meta || typeof meta.gatePassed !== "boolean") return reject("missing meta");
|
|
12309
|
+
if (!meta.gatePassed) return reject("its evaluation gate did not pass - run train-model after labeling more feedback");
|
|
12310
|
+
if (typeof meta.acceptThreshold !== "number") return reject("gate passed but no calibrated threshold (corrupt meta)");
|
|
12311
|
+
return {
|
|
12312
|
+
model: parsed.model,
|
|
12313
|
+
acceptThreshold: meta.acceptThreshold,
|
|
12314
|
+
strongThreshold: typeof meta.strongThreshold === "number" ? meta.strongThreshold : null,
|
|
12315
|
+
meta
|
|
12316
|
+
};
|
|
12317
|
+
}
|
|
12318
|
+
function saveLocalModel(fileContents, outPath) {
|
|
12319
|
+
const file = outPath ?? defaultLocalModelPath();
|
|
12320
|
+
import_node_fs2.default.mkdirSync(import_node_path2.default.dirname(file), { recursive: true });
|
|
12321
|
+
const tmp = `${file}.${process.pid}.tmp`;
|
|
12322
|
+
import_node_fs2.default.writeFileSync(tmp, JSON.stringify(fileContents, null, 2), { mode: 384 });
|
|
12323
|
+
import_node_fs2.default.renameSync(tmp, file);
|
|
12324
|
+
return file;
|
|
12325
|
+
}
|
|
12326
|
+
function readLocalModelFile(overridePath) {
|
|
12327
|
+
try {
|
|
12328
|
+
const parsed = JSON.parse(import_node_fs2.default.readFileSync(overridePath ?? defaultLocalModelPath(), "utf8"));
|
|
12329
|
+
return parsed.formatVersion === 1 && isValidNaiveBayesModel(parsed.model) ? parsed : null;
|
|
12330
|
+
} catch {
|
|
12331
|
+
return null;
|
|
12332
|
+
}
|
|
12333
|
+
}
|
|
12334
|
+
var import_node_fs2, import_node_os2, import_node_path2, testDeps2;
|
|
12335
|
+
var init_LocalModelStore = __esm({
|
|
12336
|
+
"src/analysis/LocalModelStore.ts"() {
|
|
12231
12337
|
"use strict";
|
|
12232
12338
|
init_cjs_shims();
|
|
12339
|
+
import_node_fs2 = __toESM(require("fs"));
|
|
12340
|
+
import_node_os2 = __toESM(require("os"));
|
|
12341
|
+
import_node_path2 = __toESM(require("path"));
|
|
12342
|
+
init_NaiveBayesClassifier();
|
|
12343
|
+
init_logger();
|
|
12344
|
+
testDeps2 = null;
|
|
12233
12345
|
}
|
|
12234
12346
|
});
|
|
12235
12347
|
|
|
12236
|
-
// src/analysis/
|
|
12237
|
-
function
|
|
12238
|
-
const
|
|
12239
|
-
|
|
12240
|
-
(tok) => model.labels.some((l) => model.logLikelihoods[l]?.[tok] !== void 0)
|
|
12241
|
-
);
|
|
12242
|
-
if (vocabTokens.length === 0) {
|
|
12243
|
-
return { label: "unknown", margin: 0 };
|
|
12244
|
-
}
|
|
12245
|
-
const scores = {};
|
|
12246
|
-
for (const label of model.labels) {
|
|
12247
|
-
let s = model.priors[label];
|
|
12248
|
-
const oov = Math.log(model.alpha / (model.totals[label] + model.alpha * model.vocabSize));
|
|
12249
|
-
const ll = model.logLikelihoods[label];
|
|
12250
|
-
for (const tok of vocabTokens) {
|
|
12251
|
-
s += ll[tok] ?? oov;
|
|
12252
|
-
}
|
|
12253
|
-
scores[label] = s;
|
|
12254
|
-
}
|
|
12255
|
-
const ranked = Object.entries(scores).sort((a, b) => b[1] - a[1]);
|
|
12256
|
-
const topLog = ranked[0][1];
|
|
12257
|
-
const secondLog = ranked[1]?.[1] ?? topLog;
|
|
12258
|
-
const margin = (topLog - secondLog) / vocabTokens.length;
|
|
12259
|
-
return { label: ranked[0][0], margin };
|
|
12348
|
+
// src/analysis/redact.ts
|
|
12349
|
+
function redactForStorage(raw) {
|
|
12350
|
+
const redacted = raw.replace(URI_RE, " URL ").replace(EMAIL_RE, " EMAIL ").replace(SECRET_RE, " SECRET ").replace(IP_RE, " IP ").replace(WIN_PATH_RE, " PATH ").replace(UNIX_PATH_RE, " PATH ");
|
|
12351
|
+
return tokenize(redacted).join(" ");
|
|
12260
12352
|
}
|
|
12261
|
-
|
|
12262
|
-
"
|
|
12353
|
+
function localInferenceTokens(message) {
|
|
12354
|
+
return redactForStorage(message).split(" ").filter(Boolean);
|
|
12355
|
+
}
|
|
12356
|
+
var URI_RE, EMAIL_RE, SECRET_RE, IP_RE, WIN_PATH_RE, UNIX_PATH_RE;
|
|
12357
|
+
var init_redact = __esm({
|
|
12358
|
+
"src/analysis/redact.ts"() {
|
|
12263
12359
|
"use strict";
|
|
12264
12360
|
init_cjs_shims();
|
|
12265
12361
|
init_tokenize();
|
|
12362
|
+
URI_RE = /[a-z][\w+.-]*:\/\/\S+/gi;
|
|
12363
|
+
EMAIL_RE = /\b[\w.+-]+@[\w.-]+\.\w+\b/g;
|
|
12364
|
+
SECRET_RE = /\b(?=[A-Za-z0-9_+/=-]*[0-9])(?=[A-Za-z0-9_+/=-]*[A-Za-z])[A-Za-z0-9_+/=-]{16,}\b/g;
|
|
12365
|
+
IP_RE = /\b\d{1,3}(?:\.\d{1,3}){3}\b/g;
|
|
12366
|
+
WIN_PATH_RE = /[A-Za-z]:\\[^\s'"]+/g;
|
|
12367
|
+
UNIX_PATH_RE = /(?:\/[\w.-]+){2,}/g;
|
|
12266
12368
|
}
|
|
12267
12369
|
});
|
|
12268
12370
|
|
|
12269
12371
|
// src/analysis/constants.ts
|
|
12270
|
-
var STRENGTH, STRENGTH_LEVELS, OVERRIDE_MARGIN;
|
|
12372
|
+
var CATEGORIES, STRENGTH, STRENGTH_LEVELS, OVERRIDE_MARGIN, LOCAL_TRAINING;
|
|
12271
12373
|
var init_constants = __esm({
|
|
12272
12374
|
"src/analysis/constants.ts"() {
|
|
12273
12375
|
"use strict";
|
|
12274
12376
|
init_cjs_shims();
|
|
12377
|
+
CATEGORIES = [
|
|
12378
|
+
"assertion",
|
|
12379
|
+
"env-config",
|
|
12380
|
+
"flaky-by-retry",
|
|
12381
|
+
"locator-not-found",
|
|
12382
|
+
"navigation",
|
|
12383
|
+
"network",
|
|
12384
|
+
"timeout"
|
|
12385
|
+
];
|
|
12275
12386
|
STRENGTH = {
|
|
12276
12387
|
unknownMaxMargin: 0.05,
|
|
12277
12388
|
collectMaxMargin: 0.1,
|
|
@@ -12281,6 +12392,23 @@ var init_constants = __esm({
|
|
|
12281
12392
|
};
|
|
12282
12393
|
STRENGTH_LEVELS = ["weak", "moderate", "strong"];
|
|
12283
12394
|
OVERRIDE_MARGIN = 999;
|
|
12395
|
+
LOCAL_TRAINING = {
|
|
12396
|
+
/** NB priors from fewer docs are noise, and LOOCV estimates below this are junk. */
|
|
12397
|
+
minLabeled: 10,
|
|
12398
|
+
/** A margin needs a runner-up label to exist at all. */
|
|
12399
|
+
minClasses: 2,
|
|
12400
|
+
/** Precision estimated on fewer accepted predictions is noise — refuse to calibrate. */
|
|
12401
|
+
minAccepted: 5,
|
|
12402
|
+
/** The local layer only ANSWERS when its held-out precision at the chosen threshold hits this. */
|
|
12403
|
+
acceptPrecision: 0.9,
|
|
12404
|
+
/** Bar for the local layer to render as 'strong' (else capped at 'moderate'). */
|
|
12405
|
+
strongPrecision: 0.95,
|
|
12406
|
+
/** Below this n, leave-one-out (5-fold folds would be too small to train on). */
|
|
12407
|
+
cvSwitchoverN: 50,
|
|
12408
|
+
folds: 5,
|
|
12409
|
+
/** Fixed PRNG seed — evaluation must be reproducible run-to-run. */
|
|
12410
|
+
seed: 1234
|
|
12411
|
+
};
|
|
12284
12412
|
}
|
|
12285
12413
|
});
|
|
12286
12414
|
|
|
@@ -12301,7 +12429,7 @@ function errorHeader(message) {
|
|
|
12301
12429
|
return beforeCallLog.split("\n").filter((line) => !/^\s*[-+]?\s*(?:Expected|Received)\b/i.test(line)).join("\n");
|
|
12302
12430
|
}
|
|
12303
12431
|
function isAssertionTimeout(message) {
|
|
12304
|
-
return /Expect "[^"]+" with timeout/.test(message) && /resolved to [1-9]\d* element/.test(message);
|
|
12432
|
+
return /(?:Expect "[^"]+"|expect\.\w+) with timeout/.test(message) && /resolved to [1-9]\d* element/.test(message);
|
|
12305
12433
|
}
|
|
12306
12434
|
function isNetwork(message) {
|
|
12307
12435
|
return /net::ERR_(CONNECTION|NAME_NOT_RESOLVED|INTERNET_DISCONNECTED|SSL|TIMED_OUT|ADDRESS|NETWORK_CHANGED|EMPTY_RESPONSE)|\bECONN(?:REFUSED|RESET)\b|\bENOTFOUND\b|\bEAI_AGAIN\b|getaddrinfo|socket hang up|fetch failed/.test(errorHeader(message));
|
|
@@ -12332,26 +12460,37 @@ function topFrame(stack) {
|
|
|
12332
12460
|
function representativeMessage(message) {
|
|
12333
12461
|
return clampWithEllipsis(message.split("\n")[0], MAX_MESSAGE_CHARS);
|
|
12334
12462
|
}
|
|
12335
|
-
function classifyOne(f,
|
|
12463
|
+
function classifyOne(f, chain) {
|
|
12336
12464
|
const frame = topFrame(f.error.stack);
|
|
12337
12465
|
if (f.retryHistory.some((r) => r.status === "passed")) {
|
|
12338
|
-
return { category: "flaky-by-retry", margin: OVERRIDE_MARGIN, failure: f, frame };
|
|
12466
|
+
return { category: "flaky-by-retry", margin: OVERRIDE_MARGIN, strength: "strong", source: "retry", failure: f, frame };
|
|
12339
12467
|
}
|
|
12340
12468
|
for (const [match, category] of MESSAGE_RULES) {
|
|
12341
|
-
if (match(f.error.message))
|
|
12469
|
+
if (match(f.error.message)) {
|
|
12470
|
+
return { category, margin: OVERRIDE_MARGIN, strength: "strong", source: "rule", failure: f, frame };
|
|
12471
|
+
}
|
|
12342
12472
|
}
|
|
12343
|
-
|
|
12344
|
-
|
|
12473
|
+
if (chain.local) {
|
|
12474
|
+
const { model, acceptThreshold, strongThreshold } = chain.local;
|
|
12475
|
+
const r = classifyTokens(model, localInferenceTokens(f.error.message));
|
|
12476
|
+
if (r.label !== "unknown" && r.margin >= acceptThreshold) {
|
|
12477
|
+
const strength = strongThreshold !== null && r.margin >= strongThreshold ? "strong" : "moderate";
|
|
12478
|
+
return { category: r.label, margin: r.margin, strength, source: "local", failure: f, frame };
|
|
12479
|
+
}
|
|
12480
|
+
}
|
|
12481
|
+
const { label, margin } = classify(chain.base, classifyInput(f));
|
|
12482
|
+
return { category: label, margin, strength: strengthOf(margin), source: "base", failure: f, frame };
|
|
12345
12483
|
}
|
|
12346
|
-
function classifyAll(failures,
|
|
12347
|
-
return failures.map((f) => classifyOne(f,
|
|
12484
|
+
function classifyAll(failures, chain) {
|
|
12485
|
+
return failures.map((f) => classifyOne(f, chain));
|
|
12348
12486
|
}
|
|
12349
12487
|
function perTestClassifications(classified) {
|
|
12350
12488
|
return classified.map((c) => ({
|
|
12351
12489
|
testId: c.failure.testId,
|
|
12352
12490
|
category: c.category,
|
|
12353
12491
|
message: representativeMessage(c.failure.error.message),
|
|
12354
|
-
strength:
|
|
12492
|
+
strength: c.strength
|
|
12493
|
+
// resolved at classification time, per producing source
|
|
12355
12494
|
}));
|
|
12356
12495
|
}
|
|
12357
12496
|
function clusterClassified(classified) {
|
|
@@ -12362,13 +12501,16 @@ function clusterClassified(classified) {
|
|
|
12362
12501
|
if (bucket) bucket.push(c);
|
|
12363
12502
|
else groups.set(key, [c]);
|
|
12364
12503
|
}
|
|
12504
|
+
const rank = Object.fromEntries(
|
|
12505
|
+
STRENGTH_LEVELS.map((level, i) => [level, i])
|
|
12506
|
+
);
|
|
12365
12507
|
const clusters = [];
|
|
12366
12508
|
for (const members of groups.values()) {
|
|
12367
|
-
const
|
|
12509
|
+
const strongest = members.reduce((a, b) => rank[b.strength] > rank[a.strength] ? b : a);
|
|
12368
12510
|
clusters.push({
|
|
12369
12511
|
category: members[0].category,
|
|
12370
|
-
strength:
|
|
12371
|
-
margin,
|
|
12512
|
+
strength: strongest.strength,
|
|
12513
|
+
margin: Math.max(...members.map((m) => m.margin)),
|
|
12372
12514
|
count: members.length,
|
|
12373
12515
|
tests: members.slice(0, MAX_TESTS_PER_CLUSTER).map((m) => m.failure.testTitle),
|
|
12374
12516
|
testsTotal: members.length,
|
|
@@ -12384,6 +12526,7 @@ var init_FailureClusterer = __esm({
|
|
|
12384
12526
|
"use strict";
|
|
12385
12527
|
init_cjs_shims();
|
|
12386
12528
|
init_NaiveBayesClassifier();
|
|
12529
|
+
init_redact();
|
|
12387
12530
|
init_constants();
|
|
12388
12531
|
init_truncate();
|
|
12389
12532
|
MAX_TESTS_PER_CLUSTER = 20;
|
|
@@ -12399,100 +12542,209 @@ var init_FailureClusterer = __esm({
|
|
|
12399
12542
|
|
|
12400
12543
|
// src/analysis/feedback-csv.ts
|
|
12401
12544
|
function parseFeedbackRows(content) {
|
|
12402
|
-
|
|
12545
|
+
const lines = content.trim().split("\n").filter(Boolean);
|
|
12546
|
+
if (lines.length === 0) return [];
|
|
12547
|
+
const header = lines[0].trim();
|
|
12548
|
+
const isV2 = header === FEEDBACK_HEADER;
|
|
12549
|
+
if (!isV2 && header !== LEGACY_FEEDBACK_HEADER) return [];
|
|
12550
|
+
const minParts = isV2 ? 4 : 3;
|
|
12551
|
+
const rows = [];
|
|
12552
|
+
for (const line of lines.slice(1)) {
|
|
12403
12553
|
const parts = line.split(",");
|
|
12404
|
-
|
|
12405
|
-
|
|
12554
|
+
if (parts.length < minParts) continue;
|
|
12555
|
+
const row = isV2 ? { category: parts[0], margin: parts[1], label: parts[2], text: parts.slice(3).join(",") } : { category: parts[0], margin: parts[1], label: "", text: parts.slice(2).join(",") };
|
|
12556
|
+
if (row.text.trim().length === 0) continue;
|
|
12557
|
+
rows.push(row);
|
|
12558
|
+
}
|
|
12559
|
+
return rows;
|
|
12560
|
+
}
|
|
12561
|
+
function serializeFeedbackRow(r) {
|
|
12562
|
+
const label = r.label.replace(/[,\r\n]/g, "").trim();
|
|
12563
|
+
return `${r.category},${r.margin},${label},${r.text}`;
|
|
12406
12564
|
}
|
|
12407
12565
|
function feedbackIdentity(category, text) {
|
|
12408
12566
|
return `${category} ${text}`;
|
|
12409
12567
|
}
|
|
12410
|
-
|
|
12568
|
+
function isTrainableLabel(label) {
|
|
12569
|
+
return CATEGORIES.includes(label);
|
|
12570
|
+
}
|
|
12571
|
+
var FEEDBACK_HEADER, LEGACY_FEEDBACK_HEADER;
|
|
12411
12572
|
var init_feedback_csv = __esm({
|
|
12412
12573
|
"src/analysis/feedback-csv.ts"() {
|
|
12413
12574
|
"use strict";
|
|
12414
12575
|
init_cjs_shims();
|
|
12415
|
-
|
|
12576
|
+
init_constants();
|
|
12577
|
+
FEEDBACK_HEADER = "category,margin,label,text";
|
|
12578
|
+
LEGACY_FEEDBACK_HEADER = "category,margin,text";
|
|
12416
12579
|
}
|
|
12417
12580
|
});
|
|
12418
12581
|
|
|
12419
|
-
// src/analysis/
|
|
12420
|
-
function
|
|
12421
|
-
const
|
|
12422
|
-
|
|
12582
|
+
// src/analysis/FeedbackStore.ts
|
|
12583
|
+
function readAllFeedback(home = import_node_os3.default.homedir()) {
|
|
12584
|
+
const root = import_node_path3.default.join(home, ".reportforge");
|
|
12585
|
+
if (!import_node_fs3.default.existsSync(root)) return [];
|
|
12586
|
+
const out = [];
|
|
12587
|
+
for (const entry of import_node_fs3.default.readdirSync(root).sort()) {
|
|
12588
|
+
const dir = import_node_path3.default.join(root, entry);
|
|
12589
|
+
const file = import_node_path3.default.join(dir, FILENAME);
|
|
12590
|
+
if (!import_node_fs3.default.existsSync(file)) continue;
|
|
12591
|
+
out.push({
|
|
12592
|
+
file,
|
|
12593
|
+
rows: parseFeedbackRows(import_node_fs3.default.readFileSync(file, "utf8")),
|
|
12594
|
+
project: readProjectMarker(dir)
|
|
12595
|
+
});
|
|
12596
|
+
}
|
|
12597
|
+
return out;
|
|
12598
|
+
}
|
|
12599
|
+
function readProjectMarker(projectDir) {
|
|
12600
|
+
try {
|
|
12601
|
+
const parsed = JSON.parse(import_node_fs3.default.readFileSync(import_node_path3.default.join(projectDir, MARKER_FILENAME), "utf8"));
|
|
12602
|
+
return typeof parsed?.cwd === "string" ? parsed : null;
|
|
12603
|
+
} catch {
|
|
12604
|
+
return null;
|
|
12605
|
+
}
|
|
12423
12606
|
}
|
|
12424
|
-
function
|
|
12425
|
-
|
|
12607
|
+
function writeProjectMarker(projectDir, marker) {
|
|
12608
|
+
try {
|
|
12609
|
+
atomicWrite(import_node_path3.default.join(projectDir, MARKER_FILENAME), JSON.stringify(marker));
|
|
12610
|
+
} catch {
|
|
12611
|
+
}
|
|
12612
|
+
}
|
|
12613
|
+
function labeledSamples(files) {
|
|
12614
|
+
const samples = [];
|
|
12615
|
+
let invalidLabelCount = 0;
|
|
12616
|
+
let unlabeledCount = 0;
|
|
12617
|
+
for (const f of files) {
|
|
12618
|
+
for (const r of f.rows) {
|
|
12619
|
+
if (r.label === "") {
|
|
12620
|
+
unlabeledCount++;
|
|
12621
|
+
continue;
|
|
12622
|
+
}
|
|
12623
|
+
if (!isTrainableLabel(r.label)) {
|
|
12624
|
+
invalidLabelCount++;
|
|
12625
|
+
continue;
|
|
12626
|
+
}
|
|
12627
|
+
const tokens = r.text.split(" ").filter(Boolean);
|
|
12628
|
+
if (tokens.length === 0) continue;
|
|
12629
|
+
samples.push({ label: r.label, tokens });
|
|
12630
|
+
}
|
|
12631
|
+
}
|
|
12632
|
+
return { samples, invalidLabelCount, unlabeledCount };
|
|
12633
|
+
}
|
|
12634
|
+
function rewriteFeedbackFile(file, rows) {
|
|
12635
|
+
const body = rows.map(serializeFeedbackRow).join("\n");
|
|
12636
|
+
atomicWrite(file, `${FEEDBACK_HEADER}
|
|
12637
|
+
${body}${body ? "\n" : ""}`);
|
|
12638
|
+
}
|
|
12639
|
+
function atomicWrite(file, content) {
|
|
12640
|
+
import_node_fs3.default.mkdirSync(import_node_path3.default.dirname(file), { recursive: true });
|
|
12641
|
+
const tmp = `${file}.${process.pid}.tmp`;
|
|
12642
|
+
import_node_fs3.default.writeFileSync(tmp, content, "utf8");
|
|
12643
|
+
import_node_fs3.default.renameSync(tmp, file);
|
|
12644
|
+
}
|
|
12645
|
+
var import_node_fs3, import_node_os3, import_node_path3, FILENAME, MARKER_FILENAME;
|
|
12646
|
+
var init_FeedbackStore = __esm({
|
|
12647
|
+
"src/analysis/FeedbackStore.ts"() {
|
|
12648
|
+
"use strict";
|
|
12649
|
+
init_cjs_shims();
|
|
12650
|
+
import_node_fs3 = __toESM(require("fs"));
|
|
12651
|
+
import_node_os3 = __toESM(require("os"));
|
|
12652
|
+
import_node_path3 = __toESM(require("path"));
|
|
12653
|
+
init_feedback_csv();
|
|
12654
|
+
FILENAME = "unclassified.csv";
|
|
12655
|
+
MARKER_FILENAME = "project.json";
|
|
12656
|
+
}
|
|
12657
|
+
});
|
|
12658
|
+
|
|
12659
|
+
// src/analysis/UnclassifiedCollector.ts
|
|
12660
|
+
function shouldCollect(c, scope) {
|
|
12661
|
+
if (c.source !== "base") return false;
|
|
12662
|
+
if (c.category === "unknown" || c.margin <= STRENGTH.collectMaxMargin) return true;
|
|
12663
|
+
return scope === "all";
|
|
12426
12664
|
}
|
|
12427
12665
|
function tildeify(p) {
|
|
12428
|
-
const home =
|
|
12666
|
+
const home = import_node_os4.default.homedir();
|
|
12429
12667
|
return p.startsWith(home) ? `~${p.slice(home.length)}` : p;
|
|
12430
12668
|
}
|
|
12431
|
-
function collectUnclassified(classified, projectDir) {
|
|
12432
|
-
const
|
|
12669
|
+
function collectUnclassified(classified, projectDir, opts = {}) {
|
|
12670
|
+
const scope = opts.scope ?? "blind-spots";
|
|
12671
|
+
const rows = classified.filter((c) => shouldCollect(c, scope)).map((c) => ({ text: redactForStorage(c.failure.error.message), category: c.category, margin: c.margin })).filter((r) => r.text.length > 0);
|
|
12433
12672
|
if (rows.length === 0) return;
|
|
12434
|
-
const file =
|
|
12435
|
-
const firstTime = !
|
|
12673
|
+
const file = import_node_path4.default.join(projectDir, FILENAME2);
|
|
12674
|
+
const firstTime = !import_node_fs4.default.existsSync(file);
|
|
12436
12675
|
const byIdentity = /* @__PURE__ */ new Map();
|
|
12437
12676
|
if (!firstTime) {
|
|
12438
|
-
for (const r of parseFeedbackRows(
|
|
12439
|
-
byIdentity.set(feedbackIdentity(r.category, r.text), r
|
|
12677
|
+
for (const r of parseFeedbackRows(import_node_fs4.default.readFileSync(file, "utf8"))) {
|
|
12678
|
+
byIdentity.set(feedbackIdentity(r.category, r.text), r);
|
|
12440
12679
|
}
|
|
12441
12680
|
}
|
|
12442
|
-
let added = false;
|
|
12443
12681
|
for (const r of rows) {
|
|
12444
12682
|
const id = feedbackIdentity(r.category, r.text);
|
|
12683
|
+
const label = byIdentity.get(id)?.label ?? "";
|
|
12445
12684
|
byIdentity.delete(id);
|
|
12446
|
-
byIdentity.set(id,
|
|
12447
|
-
|
|
12448
|
-
|
|
12449
|
-
|
|
12450
|
-
|
|
12451
|
-
|
|
12452
|
-
|
|
12453
|
-
|
|
12454
|
-
`, "utf8");
|
|
12685
|
+
byIdentity.set(id, { category: r.category, margin: String(r.margin), label, text: r.text });
|
|
12686
|
+
}
|
|
12687
|
+
const all = evictOldestUnlabeledFirst([...byIdentity.values()], MAX_ROWS);
|
|
12688
|
+
import_node_fs4.default.mkdirSync(projectDir, { recursive: true });
|
|
12689
|
+
rewriteFeedbackFile(file, all);
|
|
12690
|
+
if (opts.project) {
|
|
12691
|
+
writeProjectMarker(projectDir, { cwd: opts.project.cwd, outputFile: opts.project.outputFile, updatedAt: Date.now() });
|
|
12692
|
+
}
|
|
12455
12693
|
if (firstTime) {
|
|
12456
12694
|
logger.info(
|
|
12457
12695
|
`Unrecognized failures collected locally at ${tildeify(file)} to improve offline classification. Local only - never sent anywhere. Disable: failureAnalysis.collectUnclassified=false`
|
|
12458
12696
|
);
|
|
12459
12697
|
}
|
|
12460
12698
|
}
|
|
12461
|
-
|
|
12699
|
+
function evictOldestUnlabeledFirst(rows, max) {
|
|
12700
|
+
const excess = rows.length - max;
|
|
12701
|
+
if (excess <= 0) return rows;
|
|
12702
|
+
let unlabeledToDrop = Math.min(excess, rows.filter((r) => r.label === "").length);
|
|
12703
|
+
let labeledToDrop = excess - unlabeledToDrop;
|
|
12704
|
+
const out = [];
|
|
12705
|
+
for (const r of rows) {
|
|
12706
|
+
if (r.label === "" && unlabeledToDrop > 0) {
|
|
12707
|
+
unlabeledToDrop--;
|
|
12708
|
+
continue;
|
|
12709
|
+
}
|
|
12710
|
+
if (r.label !== "" && labeledToDrop > 0) {
|
|
12711
|
+
labeledToDrop--;
|
|
12712
|
+
continue;
|
|
12713
|
+
}
|
|
12714
|
+
out.push(r);
|
|
12715
|
+
}
|
|
12716
|
+
return out;
|
|
12717
|
+
}
|
|
12718
|
+
var import_node_fs4, import_node_os4, import_node_path4, MAX_ROWS, FILENAME2;
|
|
12462
12719
|
var init_UnclassifiedCollector = __esm({
|
|
12463
12720
|
"src/analysis/UnclassifiedCollector.ts"() {
|
|
12464
12721
|
"use strict";
|
|
12465
12722
|
init_cjs_shims();
|
|
12466
|
-
|
|
12467
|
-
|
|
12468
|
-
|
|
12469
|
-
init_tokenize();
|
|
12723
|
+
import_node_fs4 = __toESM(require("fs"));
|
|
12724
|
+
import_node_os4 = __toESM(require("os"));
|
|
12725
|
+
import_node_path4 = __toESM(require("path"));
|
|
12470
12726
|
init_constants();
|
|
12471
12727
|
init_feedback_csv();
|
|
12728
|
+
init_redact();
|
|
12729
|
+
init_FeedbackStore();
|
|
12472
12730
|
init_logger();
|
|
12473
12731
|
MAX_ROWS = 500;
|
|
12474
|
-
|
|
12475
|
-
URI_RE = /[a-z][\w+.-]*:\/\/\S+/gi;
|
|
12476
|
-
EMAIL_RE = /\b[\w.+-]+@[\w.-]+\.\w+\b/g;
|
|
12477
|
-
SECRET_RE = /\b(?=[A-Za-z0-9_+/=-]*[0-9])(?=[A-Za-z0-9_+/=-]*[A-Za-z])[A-Za-z0-9_+/=-]{16,}\b/g;
|
|
12478
|
-
IP_RE = /\b\d{1,3}(?:\.\d{1,3}){3}\b/g;
|
|
12479
|
-
WIN_PATH_RE = /[A-Za-z]:\\[^\s'"]+/g;
|
|
12480
|
-
UNIX_PATH_RE = /(?:\/[\w.-]+){2,}/g;
|
|
12732
|
+
FILENAME2 = "unclassified.csv";
|
|
12481
12733
|
}
|
|
12482
12734
|
});
|
|
12483
12735
|
|
|
12484
12736
|
// src/utils/paths.ts
|
|
12485
12737
|
function resolveProjectDir(options, cwd) {
|
|
12486
12738
|
const projectKey = import_node_crypto2.default.createHash("sha256").update(cwd + (options.outputFile ?? "")).digest("hex").slice(0, 8);
|
|
12487
|
-
return
|
|
12739
|
+
return import_node_path5.default.join(import_node_os5.default.homedir(), ".reportforge", projectKey);
|
|
12488
12740
|
}
|
|
12489
|
-
var
|
|
12741
|
+
var import_node_os5, import_node_path5, import_node_crypto2;
|
|
12490
12742
|
var init_paths = __esm({
|
|
12491
12743
|
"src/utils/paths.ts"() {
|
|
12492
12744
|
"use strict";
|
|
12493
12745
|
init_cjs_shims();
|
|
12494
|
-
|
|
12495
|
-
|
|
12746
|
+
import_node_os5 = __toESM(require("os"));
|
|
12747
|
+
import_node_path5 = __toESM(require("path"));
|
|
12496
12748
|
import_node_crypto2 = __toESM(require("crypto"));
|
|
12497
12749
|
}
|
|
12498
12750
|
});
|
|
@@ -12525,27 +12777,49 @@ function runFailureAnalysis(reportData, options, cwd) {
|
|
|
12525
12777
|
if (reportData.stats.failed === 0 || !fa.enabled) return;
|
|
12526
12778
|
const cap = fa.maxFailuresToAnalyse;
|
|
12527
12779
|
const analysed = reportData.failures.slice(0, cap);
|
|
12528
|
-
const
|
|
12529
|
-
const
|
|
12780
|
+
const base = loadModel(fa.autoUpdateModel);
|
|
12781
|
+
const local = fa.localModel ? loadLocalModel(fa.localModelPath) ?? void 0 : void 0;
|
|
12782
|
+
if (local) {
|
|
12783
|
+
logger.debug(
|
|
12784
|
+
`local model active (trained ${local.meta.trainedAt}, ${local.meta.labeledCount} labeled rows, accept margin >= ${local.acceptThreshold})`
|
|
12785
|
+
);
|
|
12786
|
+
if (local.meta.baseVersionAtEval !== base.version) {
|
|
12787
|
+
logger.debug(
|
|
12788
|
+
`local model was gate-checked against base v${local.meta.baseVersionAtEval} but base v${base.version} is active - re-run train-model to re-validate against the current base`
|
|
12789
|
+
);
|
|
12790
|
+
}
|
|
12791
|
+
}
|
|
12792
|
+
const classified = classifyAll(analysed, { base, local });
|
|
12793
|
+
logger.debug(`failure analysis: ${classified.length} classified - ${sourceTally(classified)}`);
|
|
12530
12794
|
reportData.classifications = perTestClassifications(classified);
|
|
12531
12795
|
if (templatesConsumeAnalysis(options)) {
|
|
12532
12796
|
reportData.analysis = analyseClassified(classified, reportData.failures.length, fa);
|
|
12533
12797
|
}
|
|
12534
12798
|
if (fa.collectUnclassified) {
|
|
12535
|
-
collectUnclassified(classified, resolveProjectDir(options, cwd)
|
|
12799
|
+
collectUnclassified(classified, resolveProjectDir(options, cwd), {
|
|
12800
|
+
scope: fa.collectScope,
|
|
12801
|
+
project: { cwd, outputFile: options.outputFile }
|
|
12802
|
+
});
|
|
12536
12803
|
}
|
|
12537
12804
|
}
|
|
12805
|
+
function sourceTally(classified) {
|
|
12806
|
+
const counts = /* @__PURE__ */ new Map();
|
|
12807
|
+
for (const c of classified) counts.set(c.source, (counts.get(c.source) ?? 0) + 1);
|
|
12808
|
+
return ["rule", "retry", "local", "base"].filter((s) => counts.has(s)).map((s) => `${counts.get(s)} ${s}`).join(" / ");
|
|
12809
|
+
}
|
|
12538
12810
|
var DEFAULTS, STRENGTH_RANK;
|
|
12539
12811
|
var init_analysis = __esm({
|
|
12540
12812
|
"src/analysis/index.ts"() {
|
|
12541
12813
|
"use strict";
|
|
12542
12814
|
init_cjs_shims();
|
|
12543
12815
|
init_ModelStore();
|
|
12816
|
+
init_LocalModelStore();
|
|
12544
12817
|
init_FailureClusterer();
|
|
12545
12818
|
init_SummaryWriter();
|
|
12546
12819
|
init_constants();
|
|
12547
12820
|
init_UnclassifiedCollector();
|
|
12548
12821
|
init_paths();
|
|
12822
|
+
init_logger();
|
|
12549
12823
|
DEFAULTS = { maxClusters: 10, minStrength: "weak", maxFailuresToAnalyse: 500 };
|
|
12550
12824
|
STRENGTH_RANK = Object.fromEntries(
|
|
12551
12825
|
STRENGTH_LEVELS.map((level, i) => [level, i])
|
|
@@ -12577,7 +12851,7 @@ async function run() {
|
|
|
12577
12851
|
console.log(` Template : ${template}`);
|
|
12578
12852
|
console.log(` Output : ${outputFile}`);
|
|
12579
12853
|
const reportData = buildDemoReportData(template);
|
|
12580
|
-
const options = parseOptions({ template, outputFile, failureAnalysis: { collectUnclassified: false } });
|
|
12854
|
+
const options = parseOptions({ template, outputFile, failureAnalysis: { collectUnclassified: false, localModel: false } });
|
|
12581
12855
|
const generator = new PdfGenerator();
|
|
12582
12856
|
runFailureAnalysis(reportData, options, process.cwd());
|
|
12583
12857
|
reportData.briefSentence = buildBriefSentence(reportData.stats, reportData.analysis, reportData.charts.trend);
|
|
@@ -12641,22 +12915,12 @@ __export(export_feedback_exports, {
|
|
|
12641
12915
|
exportFeedbackCli: () => exportFeedbackCli
|
|
12642
12916
|
});
|
|
12643
12917
|
function exportFeedback(deps = {}) {
|
|
12644
|
-
const home = deps.home ??
|
|
12918
|
+
const home = deps.home ?? import_node_os6.default.homedir();
|
|
12645
12919
|
const cwd = deps.cwd ?? process.cwd();
|
|
12646
|
-
const
|
|
12647
|
-
const rows = [];
|
|
12648
|
-
if (import_node_fs3.default.existsSync(root)) {
|
|
12649
|
-
for (const entry of import_node_fs3.default.readdirSync(root)) {
|
|
12650
|
-
const file = import_node_path4.default.join(root, entry, "unclassified.csv");
|
|
12651
|
-
if (!import_node_fs3.default.existsSync(file)) continue;
|
|
12652
|
-
for (const r of parseFeedbackRows(import_node_fs3.default.readFileSync(file, "utf8"))) {
|
|
12653
|
-
rows.push(r.raw);
|
|
12654
|
-
}
|
|
12655
|
-
}
|
|
12656
|
-
}
|
|
12920
|
+
const rows = readAllFeedback(home).flatMap((f) => f.rows.map(serializeFeedbackRow));
|
|
12657
12921
|
if (rows.length === 0) return { rowCount: 0, outFile: null };
|
|
12658
|
-
const outFile =
|
|
12659
|
-
|
|
12922
|
+
const outFile = import_node_path6.default.join(cwd, "reportforge-feedback.csv");
|
|
12923
|
+
import_node_fs5.default.writeFileSync(outFile, `${FEEDBACK_HEADER}
|
|
12660
12924
|
${rows.join("\n")}
|
|
12661
12925
|
`, "utf8");
|
|
12662
12926
|
return { rowCount: rows.length, outFile };
|
|
@@ -12673,15 +12937,558 @@ Review it before sharing; there is no upload.
|
|
|
12673
12937
|
);
|
|
12674
12938
|
}
|
|
12675
12939
|
}
|
|
12676
|
-
var
|
|
12940
|
+
var import_node_fs5, import_node_os6, import_node_path6;
|
|
12677
12941
|
var init_export_feedback = __esm({
|
|
12678
12942
|
"src/cli/export-feedback.ts"() {
|
|
12679
12943
|
"use strict";
|
|
12680
12944
|
init_cjs_shims();
|
|
12681
|
-
|
|
12682
|
-
|
|
12683
|
-
|
|
12945
|
+
import_node_fs5 = __toESM(require("fs"));
|
|
12946
|
+
import_node_os6 = __toESM(require("os"));
|
|
12947
|
+
import_node_path6 = __toESM(require("path"));
|
|
12684
12948
|
init_feedback_csv();
|
|
12949
|
+
init_FeedbackStore();
|
|
12950
|
+
}
|
|
12951
|
+
});
|
|
12952
|
+
|
|
12953
|
+
// src/cli/label-feedback.ts
|
|
12954
|
+
var label_feedback_exports = {};
|
|
12955
|
+
__export(label_feedback_exports, {
|
|
12956
|
+
labelFeedback: () => labelFeedback,
|
|
12957
|
+
labelFeedbackCli: () => labelFeedbackCli
|
|
12958
|
+
});
|
|
12959
|
+
async function labelFeedback(deps) {
|
|
12960
|
+
const { io } = deps;
|
|
12961
|
+
if (!io.isTTY) {
|
|
12962
|
+
io.write(`${nonInteractiveHelp()}
|
|
12963
|
+
`);
|
|
12964
|
+
return { labeled: 0, skipped: 0, remaining: countUnlabeled(deps.home), nonInteractive: true };
|
|
12965
|
+
}
|
|
12966
|
+
const files = readAllFeedback(deps.home ?? import_node_os7.default.homedir());
|
|
12967
|
+
const menu = CATEGORIES.map((c, i) => ` ${i + 1}) ${c}`).join("\n");
|
|
12968
|
+
let labeled = 0;
|
|
12969
|
+
let skipped = 0;
|
|
12970
|
+
let quit = false;
|
|
12971
|
+
io.write(`${CONVENTIONS}
|
|
12972
|
+
|
|
12973
|
+
`);
|
|
12974
|
+
for (const f of files) {
|
|
12975
|
+
if (quit) break;
|
|
12976
|
+
for (let i = 0; i < f.rows.length && !quit; i++) {
|
|
12977
|
+
const row = f.rows[i];
|
|
12978
|
+
if (row.label !== "") continue;
|
|
12979
|
+
io.write(
|
|
12980
|
+
`
|
|
12981
|
+
project: ${f.project?.cwd ?? "(unknown - marker written on next run)"}
|
|
12982
|
+
guess: ${row.category} @ margin ${row.margin}
|
|
12983
|
+
tokens: ${row.text}
|
|
12984
|
+
${menu}
|
|
12985
|
+
s) skip q) quit
|
|
12986
|
+
`
|
|
12987
|
+
);
|
|
12988
|
+
for (; ; ) {
|
|
12989
|
+
const answer = (await io.question("label> ")).trim().toLowerCase();
|
|
12990
|
+
if (answer === "q") {
|
|
12991
|
+
quit = true;
|
|
12992
|
+
break;
|
|
12993
|
+
}
|
|
12994
|
+
if (answer === "s") {
|
|
12995
|
+
skipped++;
|
|
12996
|
+
break;
|
|
12997
|
+
}
|
|
12998
|
+
const idx = Number.parseInt(answer, 10) - 1;
|
|
12999
|
+
if (Number.isInteger(idx) && idx >= 0 && idx < CATEGORIES.length) {
|
|
13000
|
+
row.label = CATEGORIES[idx];
|
|
13001
|
+
rewriteFeedbackFile(f.file, f.rows);
|
|
13002
|
+
labeled++;
|
|
13003
|
+
break;
|
|
13004
|
+
}
|
|
13005
|
+
io.write(` enter 1-${CATEGORIES.length}, s, or q
|
|
13006
|
+
`);
|
|
13007
|
+
}
|
|
13008
|
+
}
|
|
13009
|
+
}
|
|
13010
|
+
const remaining = files.flatMap((f) => f.rows).filter((r) => r.label === "").length;
|
|
13011
|
+
io.write(
|
|
13012
|
+
`
|
|
13013
|
+
labeled ${labeled}, skipped ${skipped}, ${remaining} still unlabeled.
|
|
13014
|
+
` + (labeled > 0 ? "next: npx @reportforge/playwright-pdf train-model\n" : "")
|
|
13015
|
+
);
|
|
13016
|
+
return { labeled, skipped, remaining, nonInteractive: false };
|
|
13017
|
+
}
|
|
13018
|
+
function countUnlabeled(home) {
|
|
13019
|
+
return readAllFeedback(home ?? import_node_os7.default.homedir()).flatMap((f) => f.rows).filter((r) => r.label === "").length;
|
|
13020
|
+
}
|
|
13021
|
+
async function labelFeedbackCli() {
|
|
13022
|
+
const { createInterface } = await import("readline/promises");
|
|
13023
|
+
const rl = createInterface({ input: process.stdin, output: process.stdout });
|
|
13024
|
+
const isTTY = Boolean(process.stdin.isTTY && process.stdout.isTTY);
|
|
13025
|
+
const io = {
|
|
13026
|
+
question: (p) => rl.question(p),
|
|
13027
|
+
// Interactive chatter → stdout; the non-interactive help is an ERROR message → stderr.
|
|
13028
|
+
write: (s) => void (isTTY ? process.stdout : process.stderr).write(s),
|
|
13029
|
+
isTTY
|
|
13030
|
+
};
|
|
13031
|
+
try {
|
|
13032
|
+
const res = await labelFeedback({ io });
|
|
13033
|
+
if (res.nonInteractive) process.exitCode = 1;
|
|
13034
|
+
} finally {
|
|
13035
|
+
rl.close();
|
|
13036
|
+
}
|
|
13037
|
+
}
|
|
13038
|
+
var import_node_os7, CONVENTIONS, nonInteractiveHelp;
|
|
13039
|
+
var init_label_feedback = __esm({
|
|
13040
|
+
"src/cli/label-feedback.ts"() {
|
|
13041
|
+
"use strict";
|
|
13042
|
+
init_cjs_shims();
|
|
13043
|
+
import_node_os7 = __toESM(require("os"));
|
|
13044
|
+
init_FeedbackStore();
|
|
13045
|
+
init_constants();
|
|
13046
|
+
CONVENTIONS = [
|
|
13047
|
+
"Labeling conventions:",
|
|
13048
|
+
' - a timed-out page.goto / "Navigation timeout exceeded" is a TIMEOUT (navigation = aborted/interrupted/crashed, not slow)',
|
|
13049
|
+
' - "Test timeout ... waiting for locator" is a TIMEOUT when the element existed but the test clock ran out,',
|
|
13050
|
+
" and LOCATOR-NOT-FOUND when the element demonstrably never existed (resolved to 0 / no such element)",
|
|
13051
|
+
" - skip anything you are not sure about - a wrong label is worse than no label"
|
|
13052
|
+
].join("\n");
|
|
13053
|
+
nonInteractiveHelp = () => [
|
|
13054
|
+
"label-feedback needs an interactive terminal (TTY) - run it directly, not through a pipe or CI.",
|
|
13055
|
+
`Hand-edit alternative: open ~/.reportforge/<project>/unclassified.csv and fill the empty 'label'`,
|
|
13056
|
+
`column with one of: ${CATEGORIES.join(" | ")}`,
|
|
13057
|
+
"Then run: npx @reportforge/playwright-pdf train-model"
|
|
13058
|
+
].join("\n");
|
|
13059
|
+
}
|
|
13060
|
+
});
|
|
13061
|
+
|
|
13062
|
+
// src/analysis/LocalTrainer.ts
|
|
13063
|
+
function trainNaiveBayes(samples, alpha = 1) {
|
|
13064
|
+
if (samples.length === 0) throw new Error("trainNaiveBayes: no samples");
|
|
13065
|
+
const labels = [...new Set(samples.map((s) => s.label))].sort();
|
|
13066
|
+
const counts = new Map(labels.map((l) => [l, /* @__PURE__ */ new Map()]));
|
|
13067
|
+
const totals = Object.fromEntries(labels.map((l) => [l, 0]));
|
|
13068
|
+
const labelDocs = Object.fromEntries(labels.map((l) => [l, 0]));
|
|
13069
|
+
const vocab = /* @__PURE__ */ new Set();
|
|
13070
|
+
for (const { label, tokens } of samples) {
|
|
13071
|
+
labelDocs[label]++;
|
|
13072
|
+
const c = counts.get(label);
|
|
13073
|
+
for (const tok of tokens) {
|
|
13074
|
+
c.set(tok, (c.get(tok) ?? 0) + 1);
|
|
13075
|
+
totals[label]++;
|
|
13076
|
+
vocab.add(tok);
|
|
13077
|
+
}
|
|
13078
|
+
}
|
|
13079
|
+
const n = samples.length;
|
|
13080
|
+
const vocabSize = vocab.size;
|
|
13081
|
+
const priors = {};
|
|
13082
|
+
const logLikelihoods = {};
|
|
13083
|
+
for (const l of labels) {
|
|
13084
|
+
priors[l] = Math.log(labelDocs[l] / n);
|
|
13085
|
+
const ll = {};
|
|
13086
|
+
for (const [tok, c] of counts.get(l)) {
|
|
13087
|
+
ll[tok] = Math.log((c + alpha) / (totals[l] + alpha * vocabSize));
|
|
13088
|
+
}
|
|
13089
|
+
logLikelihoods[l] = ll;
|
|
13090
|
+
}
|
|
13091
|
+
return {
|
|
13092
|
+
labels,
|
|
13093
|
+
priors,
|
|
13094
|
+
logLikelihoods,
|
|
13095
|
+
totals,
|
|
13096
|
+
vocabSize,
|
|
13097
|
+
alpha,
|
|
13098
|
+
version: 0,
|
|
13099
|
+
playwrightVersion: "local"
|
|
13100
|
+
};
|
|
13101
|
+
}
|
|
13102
|
+
var init_LocalTrainer = __esm({
|
|
13103
|
+
"src/analysis/LocalTrainer.ts"() {
|
|
13104
|
+
"use strict";
|
|
13105
|
+
init_cjs_shims();
|
|
13106
|
+
}
|
|
13107
|
+
});
|
|
13108
|
+
|
|
13109
|
+
// src/analysis/LocalEvaluator.ts
|
|
13110
|
+
function evaluateChain(samples, base, opts = {}) {
|
|
13111
|
+
const seed = opts.seed ?? LOCAL_TRAINING.seed;
|
|
13112
|
+
const method = samples.length < LOCAL_TRAINING.cvSwitchoverN ? "loocv" : "stratified-5fold";
|
|
13113
|
+
const folds = method === "loocv" ? samples.map((_, i) => i) : stratifiedFolds(samples, LOCAL_TRAINING.folds, seed);
|
|
13114
|
+
const foldCount = method === "loocv" ? samples.length : LOCAL_TRAINING.folds;
|
|
13115
|
+
const oof = samples.map((s) => ({
|
|
13116
|
+
trueLabel: s.label,
|
|
13117
|
+
base: pick(classifyTokens(base, s.tokens)),
|
|
13118
|
+
local: { label: "unknown", margin: 0 }
|
|
13119
|
+
// filled below
|
|
13120
|
+
}));
|
|
13121
|
+
for (let k = 0; k < foldCount; k++) {
|
|
13122
|
+
const trainRows = samples.filter((_, i) => folds[i] !== k);
|
|
13123
|
+
if (trainRows.length === 0) continue;
|
|
13124
|
+
const foldModel = trainNaiveBayes(trainRows);
|
|
13125
|
+
samples.forEach((s, i) => {
|
|
13126
|
+
if (folds[i] === k) oof[i].local = pick(classifyTokens(foldModel, s.tokens));
|
|
13127
|
+
});
|
|
13128
|
+
}
|
|
13129
|
+
const reasons = [];
|
|
13130
|
+
const acceptThreshold = calibrate(oof, LOCAL_TRAINING.acceptPrecision);
|
|
13131
|
+
const strongThreshold = acceptThreshold === null ? null : calibrate(oof, LOCAL_TRAINING.strongPrecision, acceptThreshold);
|
|
13132
|
+
const classes = [...new Set(samples.map((s) => s.label))].sort();
|
|
13133
|
+
const baseMetrics = computeMetrics(oof.map((p) => ({ trueLabel: p.trueLabel, pred: p.base.label })), classes);
|
|
13134
|
+
const chainMetrics = computeMetrics(
|
|
13135
|
+
oof.map((p) => ({
|
|
13136
|
+
trueLabel: p.trueLabel,
|
|
13137
|
+
pred: acceptThreshold !== null && p.local.label !== "unknown" && p.local.margin >= acceptThreshold ? p.local.label : p.base.label
|
|
13138
|
+
})),
|
|
13139
|
+
classes
|
|
13140
|
+
);
|
|
13141
|
+
let gatePassed = false;
|
|
13142
|
+
if (acceptThreshold === null) {
|
|
13143
|
+
reasons.push(
|
|
13144
|
+
`gate FAILED: no margin threshold reaches ${LOCAL_TRAINING.acceptPrecision * 100}% precision with >= ${LOCAL_TRAINING.minAccepted} accepted held-out predictions - the local model is never confident enough to accept safely`
|
|
13145
|
+
);
|
|
13146
|
+
} else if (chainMetrics.macroF1 <= baseMetrics.macroF1) {
|
|
13147
|
+
reasons.push(
|
|
13148
|
+
`gate FAILED: chain macro-F1 ${fmt(chainMetrics.macroF1)} does not beat base ${fmt(baseMetrics.macroF1)} (tie keeps the champion)`
|
|
13149
|
+
);
|
|
13150
|
+
} else {
|
|
13151
|
+
gatePassed = true;
|
|
13152
|
+
reasons.push(
|
|
13153
|
+
`gate passed: chain macro-F1 ${fmt(chainMetrics.macroF1)} > base ${fmt(baseMetrics.macroF1)}; the local layer answers only above its calibrated margin threshold` + (strongThreshold !== null ? "" : " and never renders 'strong'")
|
|
13154
|
+
);
|
|
13155
|
+
}
|
|
13156
|
+
return {
|
|
13157
|
+
method,
|
|
13158
|
+
n: samples.length,
|
|
13159
|
+
classes,
|
|
13160
|
+
base: baseMetrics,
|
|
13161
|
+
chain: chainMetrics,
|
|
13162
|
+
acceptThreshold,
|
|
13163
|
+
strongThreshold,
|
|
13164
|
+
gatePassed,
|
|
13165
|
+
reasons,
|
|
13166
|
+
oofLocal: oof.map((p) => p.local)
|
|
13167
|
+
};
|
|
13168
|
+
}
|
|
13169
|
+
function transferThresholds(result, samples, finalModel) {
|
|
13170
|
+
const finalMarginsOver = (oofThreshold) => result.oofLocal.map((p, i) => ({ p, i })).filter(({ p }) => p.label !== "unknown" && p.margin >= oofThreshold).map(({ i }) => classifyTokens(finalModel, samples[i].tokens)).filter((r) => r.label !== "unknown").map((r) => r.margin);
|
|
13171
|
+
if (result.acceptThreshold === null) return { acceptThreshold: null, strongThreshold: null };
|
|
13172
|
+
const accept = finalMarginsOver(result.acceptThreshold);
|
|
13173
|
+
if (accept.length === 0) return { acceptThreshold: null, strongThreshold: null };
|
|
13174
|
+
const acceptThreshold = Math.min(...accept);
|
|
13175
|
+
if (result.strongThreshold === null) return { acceptThreshold, strongThreshold: null };
|
|
13176
|
+
const strong = finalMarginsOver(result.strongThreshold);
|
|
13177
|
+
return {
|
|
13178
|
+
acceptThreshold,
|
|
13179
|
+
// 'strong' must never be easier to reach than acceptance itself.
|
|
13180
|
+
strongThreshold: strong.length === 0 ? null : Math.max(Math.min(...strong), acceptThreshold)
|
|
13181
|
+
};
|
|
13182
|
+
}
|
|
13183
|
+
function calibrate(oof, target, atLeast = -Infinity) {
|
|
13184
|
+
const scored = oof.filter((p) => p.local.label !== "unknown");
|
|
13185
|
+
const candidates = [...new Set(scored.map((p) => p.local.margin))].filter((t) => t >= atLeast).sort((a, b) => a - b);
|
|
13186
|
+
for (const t of candidates) {
|
|
13187
|
+
const accepted = scored.filter((p) => p.local.margin >= t);
|
|
13188
|
+
if (accepted.length < LOCAL_TRAINING.minAccepted) break;
|
|
13189
|
+
const correct = accepted.filter((p) => p.local.label === p.trueLabel).length;
|
|
13190
|
+
if (correct / accepted.length >= target) return t;
|
|
13191
|
+
}
|
|
13192
|
+
return null;
|
|
13193
|
+
}
|
|
13194
|
+
function computeMetrics(preds, classes) {
|
|
13195
|
+
var _a;
|
|
13196
|
+
const confusion = {};
|
|
13197
|
+
for (const p of preds) {
|
|
13198
|
+
(confusion[_a = p.trueLabel] ?? (confusion[_a] = {}))[p.pred] = (confusion[p.trueLabel][p.pred] ?? 0) + 1;
|
|
13199
|
+
}
|
|
13200
|
+
const perClass = {};
|
|
13201
|
+
let f1Sum = 0;
|
|
13202
|
+
for (const c of classes) {
|
|
13203
|
+
const tp = preds.filter((p) => p.trueLabel === c && p.pred === c).length;
|
|
13204
|
+
const fp = preds.filter((p) => p.trueLabel !== c && p.pred === c).length;
|
|
13205
|
+
const fn = preds.filter((p) => p.trueLabel === c && p.pred !== c).length;
|
|
13206
|
+
const precision = tp + fp === 0 ? 0 : tp / (tp + fp);
|
|
13207
|
+
const recall = tp + fn === 0 ? 0 : tp / (tp + fn);
|
|
13208
|
+
const f1 = precision + recall === 0 ? 0 : 2 * precision * recall / (precision + recall);
|
|
13209
|
+
perClass[c] = { precision, recall, f1, support: tp + fn };
|
|
13210
|
+
f1Sum += f1;
|
|
13211
|
+
}
|
|
13212
|
+
return {
|
|
13213
|
+
accuracy: preds.length === 0 ? 0 : preds.filter((p) => p.pred === p.trueLabel).length / preds.length,
|
|
13214
|
+
macroF1: classes.length === 0 ? 0 : f1Sum / classes.length,
|
|
13215
|
+
perClass,
|
|
13216
|
+
confusion
|
|
13217
|
+
};
|
|
13218
|
+
}
|
|
13219
|
+
function stratifiedFolds(samples, folds, seed) {
|
|
13220
|
+
const rng = mulberry32(seed);
|
|
13221
|
+
const byLabel = /* @__PURE__ */ new Map();
|
|
13222
|
+
samples.forEach((s, i) => {
|
|
13223
|
+
const arr = byLabel.get(s.label);
|
|
13224
|
+
if (arr) arr.push(i);
|
|
13225
|
+
else byLabel.set(s.label, [i]);
|
|
13226
|
+
});
|
|
13227
|
+
const assignment = new Array(samples.length);
|
|
13228
|
+
for (const label of [...byLabel.keys()].sort()) {
|
|
13229
|
+
const indices = shuffle(byLabel.get(label), rng);
|
|
13230
|
+
indices.forEach((rowIdx, j) => {
|
|
13231
|
+
assignment[rowIdx] = j % folds;
|
|
13232
|
+
});
|
|
13233
|
+
}
|
|
13234
|
+
return assignment;
|
|
13235
|
+
}
|
|
13236
|
+
function shuffle(arr, rng) {
|
|
13237
|
+
const a = [...arr];
|
|
13238
|
+
for (let i = a.length - 1; i > 0; i--) {
|
|
13239
|
+
const j = Math.floor(rng() * (i + 1));
|
|
13240
|
+
[a[i], a[j]] = [a[j], a[i]];
|
|
13241
|
+
}
|
|
13242
|
+
return a;
|
|
13243
|
+
}
|
|
13244
|
+
function mulberry32(seed) {
|
|
13245
|
+
let a = seed >>> 0;
|
|
13246
|
+
return () => {
|
|
13247
|
+
a |= 0;
|
|
13248
|
+
a = a + 1831565813 | 0;
|
|
13249
|
+
let t = Math.imul(a ^ a >>> 15, 1 | a);
|
|
13250
|
+
t = t + Math.imul(t ^ t >>> 7, 61 | t) ^ t;
|
|
13251
|
+
return ((t ^ t >>> 14) >>> 0) / 4294967296;
|
|
13252
|
+
};
|
|
13253
|
+
}
|
|
13254
|
+
var pick, fmt;
|
|
13255
|
+
var init_LocalEvaluator = __esm({
|
|
13256
|
+
"src/analysis/LocalEvaluator.ts"() {
|
|
13257
|
+
"use strict";
|
|
13258
|
+
init_cjs_shims();
|
|
13259
|
+
init_NaiveBayesClassifier();
|
|
13260
|
+
init_LocalTrainer();
|
|
13261
|
+
init_constants();
|
|
13262
|
+
pick = (r) => ({ label: r.label, margin: r.margin });
|
|
13263
|
+
fmt = (x) => x.toFixed(3);
|
|
13264
|
+
}
|
|
13265
|
+
});
|
|
13266
|
+
|
|
13267
|
+
// src/cli/model-cli-shared.ts
|
|
13268
|
+
function gatherSamples(deps = {}) {
|
|
13269
|
+
const home = deps.home ?? import_node_os8.default.homedir();
|
|
13270
|
+
const cwd = deps.cwd ?? process.cwd();
|
|
13271
|
+
let files = readAllFeedback(home);
|
|
13272
|
+
const warnings = [];
|
|
13273
|
+
if (deps.projectOnly) {
|
|
13274
|
+
const unmarked = files.filter((f) => f.project === null).length;
|
|
13275
|
+
files = files.filter((f) => f.project?.cwd === cwd);
|
|
13276
|
+
if (unmarked > 0) {
|
|
13277
|
+
warnings.push(
|
|
13278
|
+
`${unmarked} feedback dir(s) have no project marker yet (written on the next reporter run) - --project cannot match them`
|
|
13279
|
+
);
|
|
13280
|
+
}
|
|
13281
|
+
}
|
|
13282
|
+
const { samples, invalidLabelCount, unlabeledCount } = labeledSamples(files);
|
|
13283
|
+
if (invalidLabelCount > 0) {
|
|
13284
|
+
warnings.push(
|
|
13285
|
+
`${invalidLabelCount} row(s) have unrecognized labels and were skipped (valid: ${CATEGORIES.join(", ")})`
|
|
13286
|
+
);
|
|
13287
|
+
}
|
|
13288
|
+
if (unlabeledCount > 0) {
|
|
13289
|
+
warnings.push(`${unlabeledCount} collected row(s) are still unlabeled - label them via label-feedback`);
|
|
13290
|
+
}
|
|
13291
|
+
const classes = new Set(samples.map((s) => s.label));
|
|
13292
|
+
let refusal = null;
|
|
13293
|
+
if (samples.length < LOCAL_TRAINING.minLabeled || classes.size < LOCAL_TRAINING.minClasses) {
|
|
13294
|
+
const tally = [...classes].sort().map((c) => `${c}: ${samples.filter((s) => s.label === c).length}`).join(", ") || "none";
|
|
13295
|
+
refusal = `not enough labeled feedback to train safely: ${samples.length} labeled row(s) across ${classes.size} class(es) (${tally}). Need >= ${LOCAL_TRAINING.minLabeled} rows across >= ${LOCAL_TRAINING.minClasses} classes - run your tests, then label collected failures via: npx @reportforge/playwright-pdf label-feedback`;
|
|
13296
|
+
}
|
|
13297
|
+
return { samples, files, warnings, refusal };
|
|
13298
|
+
}
|
|
13299
|
+
function formatEvalReport(r, baseVersion) {
|
|
13300
|
+
const lines = [];
|
|
13301
|
+
lines.push(`Evaluation: ${r.method} over ${r.n} labeled row(s), ${r.classes.length} class(es) [base model v${baseVersion}]`);
|
|
13302
|
+
lines.push("");
|
|
13303
|
+
lines.push(pad(["arm", "accuracy", "macro-F1"], [10, 10, 10]));
|
|
13304
|
+
lines.push(pad(["base", pct(r.base.accuracy), num(r.base.macroF1)], [10, 10, 10]));
|
|
13305
|
+
lines.push(pad(["chain", pct(r.chain.accuracy), num(r.chain.macroF1)], [10, 10, 10]));
|
|
13306
|
+
lines.push("");
|
|
13307
|
+
lines.push(pad(["class", "support", "base F1", "chain F1", "chain P", "chain R"], [20, 8, 9, 9, 8, 8]));
|
|
13308
|
+
for (const c of r.classes) {
|
|
13309
|
+
const b = r.base.perClass[c];
|
|
13310
|
+
const ch = r.chain.perClass[c];
|
|
13311
|
+
lines.push(pad([c, String(ch.support), num(b.f1), num(ch.f1), num(ch.precision), num(ch.recall)], [20, 8, 9, 9, 8, 8]));
|
|
13312
|
+
}
|
|
13313
|
+
lines.push("");
|
|
13314
|
+
lines.push(formatConfusion(r.chain, "chain confusion (true -> predicted)"));
|
|
13315
|
+
lines.push("");
|
|
13316
|
+
lines.push(
|
|
13317
|
+
`accept threshold: ${r.acceptThreshold === null ? "none reachable" : num(r.acceptThreshold)} strong threshold: ${r.strongThreshold === null ? "n/a" : num(r.strongThreshold)}`
|
|
13318
|
+
);
|
|
13319
|
+
for (const reason of r.reasons) lines.push(reason);
|
|
13320
|
+
return lines.join("\n");
|
|
13321
|
+
}
|
|
13322
|
+
function formatConfusion(m, title) {
|
|
13323
|
+
const predicted = [...new Set(Object.values(m.confusion).flatMap((row) => Object.keys(row)))].sort();
|
|
13324
|
+
const lines = [title, pad(["true \\ pred", ...predicted], [20, ...predicted.map(() => 12)])];
|
|
13325
|
+
for (const t of Object.keys(m.confusion).sort()) {
|
|
13326
|
+
lines.push(pad([t, ...predicted.map((p) => String(m.confusion[t][p] ?? 0))], [20, ...predicted.map(() => 12)]));
|
|
13327
|
+
}
|
|
13328
|
+
return lines.join("\n");
|
|
13329
|
+
}
|
|
13330
|
+
function pad(cells, widths) {
|
|
13331
|
+
return cells.map((c, i) => c.padEnd(widths[i])).join(" ").trimEnd();
|
|
13332
|
+
}
|
|
13333
|
+
function flagValue(argv, flag) {
|
|
13334
|
+
const hit = argv.find((a) => a.startsWith(`${flag}=`));
|
|
13335
|
+
return hit?.slice(flag.length + 1);
|
|
13336
|
+
}
|
|
13337
|
+
function hasFlag(argv, flag) {
|
|
13338
|
+
return argv.includes(flag);
|
|
13339
|
+
}
|
|
13340
|
+
var import_node_os8, pct, num;
|
|
13341
|
+
var init_model_cli_shared = __esm({
|
|
13342
|
+
"src/cli/model-cli-shared.ts"() {
|
|
13343
|
+
"use strict";
|
|
13344
|
+
init_cjs_shims();
|
|
13345
|
+
import_node_os8 = __toESM(require("os"));
|
|
13346
|
+
init_FeedbackStore();
|
|
13347
|
+
init_constants();
|
|
13348
|
+
pct = (x) => `${(x * 100).toFixed(1)}%`;
|
|
13349
|
+
num = (x) => x.toFixed(3);
|
|
13350
|
+
}
|
|
13351
|
+
});
|
|
13352
|
+
|
|
13353
|
+
// src/cli/train-model.ts
|
|
13354
|
+
var train_model_exports = {};
|
|
13355
|
+
__export(train_model_exports, {
|
|
13356
|
+
trainModel: () => trainModel,
|
|
13357
|
+
trainModelCli: () => trainModelCli
|
|
13358
|
+
});
|
|
13359
|
+
function trainModel(deps = {}) {
|
|
13360
|
+
const gathered = gatherSamples(deps);
|
|
13361
|
+
if (gathered.refusal) {
|
|
13362
|
+
return { outcome: "insufficient-data", warnings: gathered.warnings, detail: gathered.refusal };
|
|
13363
|
+
}
|
|
13364
|
+
const base = loadModel(true);
|
|
13365
|
+
const evalResult = evaluateChain(gathered.samples, base);
|
|
13366
|
+
const classes = new Set(gathered.samples.map((s) => s.label));
|
|
13367
|
+
const finalModel = trainNaiveBayes(gathered.samples);
|
|
13368
|
+
const transferred = transferThresholds(evalResult, gathered.samples, finalModel);
|
|
13369
|
+
let gatePassed = evalResult.gatePassed;
|
|
13370
|
+
if (gatePassed && transferred.acceptThreshold === null) {
|
|
13371
|
+
gatePassed = false;
|
|
13372
|
+
evalResult.reasons.push(
|
|
13373
|
+
"gate FAILED after threshold transfer: the OOF-accepted rows are not confidently classified by the full-data model"
|
|
13374
|
+
);
|
|
13375
|
+
}
|
|
13376
|
+
const file = {
|
|
13377
|
+
formatVersion: 1,
|
|
13378
|
+
model: finalModel,
|
|
13379
|
+
meta: {
|
|
13380
|
+
trainedAt: (/* @__PURE__ */ new Date()).toISOString(),
|
|
13381
|
+
labeledCount: gathered.samples.length,
|
|
13382
|
+
classCount: classes.size,
|
|
13383
|
+
baseVersionAtEval: base.version,
|
|
13384
|
+
gatePassed,
|
|
13385
|
+
acceptThreshold: transferred.acceptThreshold,
|
|
13386
|
+
strongThreshold: transferred.strongThreshold,
|
|
13387
|
+
metrics: { method: evalResult.method, base: evalResult.base, chain: evalResult.chain }
|
|
13388
|
+
}
|
|
13389
|
+
};
|
|
13390
|
+
const outFile = saveLocalModel(file, deps.outFile);
|
|
13391
|
+
evalResult.acceptThreshold = transferred.acceptThreshold ?? evalResult.acceptThreshold;
|
|
13392
|
+
evalResult.strongThreshold = transferred.strongThreshold;
|
|
13393
|
+
return {
|
|
13394
|
+
outcome: gatePassed ? "trained" : "gate-failed",
|
|
13395
|
+
evalResult,
|
|
13396
|
+
baseVersion: base.version,
|
|
13397
|
+
outFile,
|
|
13398
|
+
warnings: gathered.warnings,
|
|
13399
|
+
detail: gatePassed ? `local model ACTIVE - saved to ${outFile}` : `local model saved to ${outFile} but INACTIVE (gate failed) - the reporter stays on the base model`
|
|
13400
|
+
};
|
|
13401
|
+
}
|
|
13402
|
+
function trainModelCli(argv) {
|
|
13403
|
+
const out = (s) => process.stdout.write(`${s}
|
|
13404
|
+
`);
|
|
13405
|
+
const outFile = flagValue(argv, "--out");
|
|
13406
|
+
const projectOnly = hasFlag(argv, "--project");
|
|
13407
|
+
if (outFile !== void 0 && !projectOnly) {
|
|
13408
|
+
out(
|
|
13409
|
+
"[reportforge] note: --out without --project trains on feedback from ALL local projects. If this file will be committed to one repo, consider adding --project so other projects\u2019 token statistics do not ride along."
|
|
13410
|
+
);
|
|
13411
|
+
}
|
|
13412
|
+
const res = trainModel({ outFile, projectOnly });
|
|
13413
|
+
for (const w of res.warnings) out(`[reportforge] warn: ${w}`);
|
|
13414
|
+
if (res.outcome === "insufficient-data") {
|
|
13415
|
+
out(`[reportforge] ${res.detail}`);
|
|
13416
|
+
process.exitCode = 1;
|
|
13417
|
+
return;
|
|
13418
|
+
}
|
|
13419
|
+
out("");
|
|
13420
|
+
out(formatEvalReport(res.evalResult, res.baseVersion));
|
|
13421
|
+
out("");
|
|
13422
|
+
out(`[reportforge] ${res.detail}`);
|
|
13423
|
+
if (res.outcome === "trained" && res.outFile === defaultLocalModelPath()) {
|
|
13424
|
+
out("[reportforge] the reporter picks it up automatically on the next run (failureAnalysis.localModel).");
|
|
13425
|
+
}
|
|
13426
|
+
process.exitCode = res.outcome === "trained" ? 0 : 1;
|
|
13427
|
+
}
|
|
13428
|
+
var init_train_model = __esm({
|
|
13429
|
+
"src/cli/train-model.ts"() {
|
|
13430
|
+
"use strict";
|
|
13431
|
+
init_cjs_shims();
|
|
13432
|
+
init_LocalTrainer();
|
|
13433
|
+
init_LocalEvaluator();
|
|
13434
|
+
init_ModelStore();
|
|
13435
|
+
init_LocalModelStore();
|
|
13436
|
+
init_model_cli_shared();
|
|
13437
|
+
}
|
|
13438
|
+
});
|
|
13439
|
+
|
|
13440
|
+
// src/cli/evaluate-model.ts
|
|
13441
|
+
var evaluate_model_exports = {};
|
|
13442
|
+
__export(evaluate_model_exports, {
|
|
13443
|
+
evaluateModel: () => evaluateModel,
|
|
13444
|
+
evaluateModelCli: () => evaluateModelCli
|
|
13445
|
+
});
|
|
13446
|
+
function evaluateModel(deps = {}) {
|
|
13447
|
+
const lines = [];
|
|
13448
|
+
const base = loadModel(true);
|
|
13449
|
+
if (deps.modelPath !== void 0) {
|
|
13450
|
+
const stored = readLocalModelFile(deps.modelPath);
|
|
13451
|
+
if (!stored) {
|
|
13452
|
+
lines.push(`stored model at ${deps.modelPath}: unreadable or invalid`);
|
|
13453
|
+
} else {
|
|
13454
|
+
const m = stored.meta;
|
|
13455
|
+
lines.push(
|
|
13456
|
+
`stored model at ${deps.modelPath}: trained ${m.trainedAt}, ${m.labeledCount} rows / ${m.classCount} classes, gate ${m.gatePassed ? "PASSED" : "FAILED"}, recorded chain macro-F1 ${m.metrics.chain.macroF1.toFixed(3)} vs base ${m.metrics.base.macroF1.toFixed(3)}`
|
|
13457
|
+
);
|
|
13458
|
+
if (m.baseVersionAtEval !== base.version) {
|
|
13459
|
+
lines.push(
|
|
13460
|
+
`warn: that gate was checked against base v${m.baseVersionAtEval}; base v${base.version} is active now - re-run train-model to re-validate`
|
|
13461
|
+
);
|
|
13462
|
+
}
|
|
13463
|
+
}
|
|
13464
|
+
lines.push("");
|
|
13465
|
+
}
|
|
13466
|
+
const gathered = gatherSamples(deps);
|
|
13467
|
+
for (const w of gathered.warnings) lines.push(`warn: ${w}`);
|
|
13468
|
+
if (gathered.refusal) {
|
|
13469
|
+
lines.push(gathered.refusal);
|
|
13470
|
+
return { ok: false, lines };
|
|
13471
|
+
}
|
|
13472
|
+
lines.push(formatEvalReport(evaluateChain(gathered.samples, base), base.version));
|
|
13473
|
+
lines.push("(nothing was written - run train-model to produce/update the artifact)");
|
|
13474
|
+
return { ok: true, lines };
|
|
13475
|
+
}
|
|
13476
|
+
function evaluateModelCli(argv) {
|
|
13477
|
+
const res = evaluateModel({
|
|
13478
|
+
modelPath: flagValue(argv, "--model"),
|
|
13479
|
+
projectOnly: hasFlag(argv, "--project")
|
|
13480
|
+
});
|
|
13481
|
+
process.stdout.write(res.lines.map((l) => `[reportforge] ${l}`).join("\n") + "\n");
|
|
13482
|
+
process.exitCode = res.ok ? 0 : 1;
|
|
13483
|
+
}
|
|
13484
|
+
var init_evaluate_model = __esm({
|
|
13485
|
+
"src/cli/evaluate-model.ts"() {
|
|
13486
|
+
"use strict";
|
|
13487
|
+
init_cjs_shims();
|
|
13488
|
+
init_LocalEvaluator();
|
|
13489
|
+
init_ModelStore();
|
|
13490
|
+
init_LocalModelStore();
|
|
13491
|
+
init_model_cli_shared();
|
|
12685
13492
|
}
|
|
12686
13493
|
});
|
|
12687
13494
|
|
|
@@ -12699,6 +13506,12 @@ Commands:
|
|
|
12699
13506
|
--template=minimal|detailed|executive (default: detailed)
|
|
12700
13507
|
--output=<path> (default: playwright-report/demo-report.pdf)
|
|
12701
13508
|
export-feedback Export locally-collected unrecognized failures (tokenized, local-only).
|
|
13509
|
+
label-feedback Interactively label locally-collected failures for on-device training (local-only).
|
|
13510
|
+
train-model Train + evaluate a personal classifier layer from your labeled feedback (local-only).
|
|
13511
|
+
--out=<path> write the model file elsewhere (default: ~/.reportforge/model-local.json)
|
|
13512
|
+
--project train only on feedback from the current directory's project
|
|
13513
|
+
evaluate-model Compare base vs. base+local on your labeled feedback without writing anything.
|
|
13514
|
+
--model=<path> also print a stored model file's metadata and metrics
|
|
12702
13515
|
|
|
12703
13516
|
Run "npx @reportforge/playwright-pdf demo" to see the report ReportForge produces.`;
|
|
12704
13517
|
function pickCommand(arg) {
|
|
@@ -12707,6 +13520,12 @@ function pickCommand(arg) {
|
|
|
12707
13520
|
return "demo";
|
|
12708
13521
|
case "export-feedback":
|
|
12709
13522
|
return "export-feedback";
|
|
13523
|
+
case "label-feedback":
|
|
13524
|
+
return "label-feedback";
|
|
13525
|
+
case "train-model":
|
|
13526
|
+
return "train-model";
|
|
13527
|
+
case "evaluate-model":
|
|
13528
|
+
return "evaluate-model";
|
|
12710
13529
|
default:
|
|
12711
13530
|
return "help";
|
|
12712
13531
|
}
|
|
@@ -12740,6 +13559,21 @@ async function main() {
|
|
|
12740
13559
|
exportFeedbackCli2();
|
|
12741
13560
|
return;
|
|
12742
13561
|
}
|
|
13562
|
+
case "label-feedback": {
|
|
13563
|
+
const { labelFeedbackCli: labelFeedbackCli2 } = await Promise.resolve().then(() => (init_label_feedback(), label_feedback_exports));
|
|
13564
|
+
await labelFeedbackCli2();
|
|
13565
|
+
return;
|
|
13566
|
+
}
|
|
13567
|
+
case "train-model": {
|
|
13568
|
+
const { trainModelCli: trainModelCli2 } = await Promise.resolve().then(() => (init_train_model(), train_model_exports));
|
|
13569
|
+
trainModelCli2(process.argv.slice(3));
|
|
13570
|
+
return;
|
|
13571
|
+
}
|
|
13572
|
+
case "evaluate-model": {
|
|
13573
|
+
const { evaluateModelCli: evaluateModelCli2 } = await Promise.resolve().then(() => (init_evaluate_model(), evaluate_model_exports));
|
|
13574
|
+
evaluateModelCli2(process.argv.slice(3));
|
|
13575
|
+
return;
|
|
13576
|
+
}
|
|
12743
13577
|
case "help": {
|
|
12744
13578
|
const { text, stream, code } = helpOutput(arg);
|
|
12745
13579
|
(stream === "stdout" ? process.stdout : process.stderr).write(text);
|