@qulib/core 0.10.0 → 0.10.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/baseline/baseline.schema.d.ts +26 -26
- package/dist/baseline/baseline.schema.d.ts.map +1 -1
- package/dist/baseline/baseline.schema.js +1 -0
- package/dist/cli/confidence-run.js +5 -5
- package/dist/index.d.ts +1 -0
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +1 -0
- package/dist/phases/think.d.ts.map +1 -1
- package/dist/phases/think.js +4 -1
- package/dist/reporters/heatmap.d.ts +1 -1
- package/dist/reporters/heatmap.d.ts.map +1 -1
- package/dist/reporters/heatmap.js +2 -0
- package/dist/schemas/confidence.schema.d.ts +2 -2
- package/dist/schemas/gap-analysis.schema.d.ts +8 -8
- package/dist/schemas/gap-analysis.schema.js +1 -1
- package/dist/schemas/golden-manifest.schema.d.ts +137 -0
- package/dist/schemas/golden-manifest.schema.d.ts.map +1 -0
- package/dist/schemas/golden-manifest.schema.js +25 -0
- package/dist/schemas/index.d.ts +1 -0
- package/dist/schemas/index.d.ts.map +1 -1
- package/dist/schemas/index.js +1 -0
- package/dist/schemas/public-surface.schema.d.ts +15 -5
- package/dist/schemas/public-surface.schema.d.ts.map +1 -1
- package/dist/schemas/route-inventory.schema.d.ts +20 -0
- package/dist/schemas/route-inventory.schema.d.ts.map +1 -1
- package/dist/schemas/route-inventory.schema.js +4 -0
- package/dist/schemas/views.schema.d.ts +1 -1
- package/dist/tools/scoring/confidence.d.ts.map +1 -1
- package/dist/tools/scoring/confidence.js +140 -14
- package/dist/tools/scoring/prompt-leakage.d.ts +29 -0
- package/dist/tools/scoring/prompt-leakage.d.ts.map +1 -0
- package/dist/tools/scoring/prompt-leakage.js +256 -0
- package/package.json +2 -2
package/dist/schemas/index.d.ts
CHANGED
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
export { GoldenManifestSchema, GoldenSiteSchema, GoldenSiteExpectedSchema, type GoldenManifest, type GoldenSite, type GoldenSiteExpected, } from './golden-manifest.schema.js';
|
|
1
2
|
export { HarnessConfigSchema, resolveMaxOutputTokensPerLlmCall, AuthConfigSchema, DetectedAuthSchema, AuthPathRequirementsSchema, AuthPathSchema, AuthExplorationSchema, type ExplorerType, type AdapterType, type FormLoginAuthConfig, type StorageStateAuthConfig, type AuthConfig, type HarnessConfig, type DetectedAuth, type AuthPathRequirements, type AuthPath, type AuthExploration, } from './config.schema.js';
|
|
2
3
|
export { DecisionLogEntrySchema, type DecisionLogEntry, } from './decision-log.schema.js';
|
|
3
4
|
export { RouteInventorySchema, RouteSchema, A11yViolationSchema, BrokenLinkSchema, type RouteInventory, type Route, } from './route-inventory.schema.js';
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/schemas/index.ts"],"names":[],"mappings":"AAAA,OAAO,EACL,mBAAmB,EACnB,gCAAgC,EAChC,gBAAgB,EAChB,kBAAkB,EAClB,0BAA0B,EAC1B,cAAc,EACd,qBAAqB,EACrB,KAAK,YAAY,EACjB,KAAK,WAAW,EAChB,KAAK,mBAAmB,EACxB,KAAK,sBAAsB,EAC3B,KAAK,UAAU,EACf,KAAK,aAAa,EAClB,KAAK,YAAY,EACjB,KAAK,oBAAoB,EACzB,KAAK,QAAQ,EACb,KAAK,eAAe,GACrB,MAAM,oBAAoB,CAAC;AAC5B,OAAO,EACL,sBAAsB,EACtB,KAAK,gBAAgB,GACtB,MAAM,0BAA0B,CAAC;AAClC,OAAO,EACL,oBAAoB,EACpB,WAAW,EACX,mBAAmB,EACnB,gBAAgB,EAChB,KAAK,cAAc,EACnB,KAAK,KAAK,GACX,MAAM,6BAA6B,CAAC;AACrC,OAAO,EACL,iBAAiB,EACjB,SAAS,EACT,qBAAqB,EACrB,mBAAmB,EACnB,cAAc,EACd,6BAA6B,EAC7B,KAAK,WAAW,EAChB,KAAK,GAAG,EACR,KAAK,eAAe,EACpB,KAAK,aAAa,EAClB,KAAK,QAAQ,EACb,KAAK,uBAAuB,GAC7B,MAAM,0BAA0B,CAAC;AAClC,OAAO,EACL,sBAAsB,EACtB,oBAAoB,EACpB,oBAAoB,EACpB,sBAAsB,EACtB,uBAAuB,EACvB,2BAA2B,EAC3B,KAAK,gBAAgB,EACrB,KAAK,cAAc,EACnB,KAAK,cAAc,EACnB,KAAK,gBAAgB,EACrB,KAAK,iBAAiB,EACtB,KAAK,qBAAqB,GAC3B,MAAM,+BAA+B,CAAC;AACvC,OAAO,EACL,kBAAkB,EAClB,wBAAwB,EACxB,8BAA8B,EAC9B,kCAAkC,EAClC,2BAA2B,EAC3B,KAAK,YAAY,EACjB,KAAK,wBAAwB,EAC7B,KAAK,wBAAwB,GAC9B,MAAM,2BAA2B,CAAC;AACnC,OAAO,EACL,wBAAwB,EACxB,iCAAiC,EACjC,qCAAqC,EACrC,KAAK,kBAAkB,EACvB,KAAK,2BAA2B,EAChC,KAAK,+BAA+B,GACrC,MAAM,iCAAiC,CAAC;AACzC,OAAO,EACL,mBAAmB,EACnB,4BAA4B,EAC5B,6BAA6B,EAC7B,KAAK,aAAa,EAClB,KAAK,sBAAsB,EAC3B,KAAK,uBAAuB,GAC7B,MAAM,4BAA4B,CAAC;AACpC,OAAO,EACL,cAAc,EACd,kBAAkB,EAClB,KAAK,QAAQ,EACb,KAAK,YAAY,GAClB,MAAM,oBAAoB,CAAC;AAC5B,OAAO,EACL,wBAAwB,EACxB,kBAAkB,EAClB,uBAAuB,EACvB,qBAAqB,EACrB,sBAAsB,EACtB,uBAAuB,EACvB,4BAA4B,EAC5B,uBAAuB,EACvB,KAAK,kBAAkB,EACvB,KAAK,YAAY,EACjB,KAAK,iBAAiB,EACtB,KAAK,eAAe,EACpB,KAAK,gBAAgB,EACrB,KAAK,iBAAiB,EACtB,KAAK,sBAAsB,EAC3B,KAAK,iBAAiB,GACvB,MAAM,wBAAwB,CAAC;AAChC,OAAO,EACL,0BAA0B,EAC1B,mBAAmB,EACnB,eAAe,EACf,gBAAgB,EAChB,iBAAiB,EACjB,gBAAgB,EAChB,KAAK,oBAAoB,EACzB,KAAK,aAAa,EAClB,KAAK,SAAS,EACd,KAAK,UAAU,EACf,KAAK,WAAW,EAChB,KAAK,UAAU,GAChB,MAAM,mBAAmB,CAAC"}
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/schemas/index.ts"],"names":[],"mappings":"AAAA,OAAO,EACL,oBAAoB,EACpB,gBAAgB,EAChB,wBAAwB,EACxB,KAAK,cAAc,EACnB,KAAK,UAAU,EACf,KAAK,kBAAkB,GACxB,MAAM,6BAA6B,CAAC;AACrC,OAAO,EACL,mBAAmB,EACnB,gCAAgC,EAChC,gBAAgB,EAChB,kBAAkB,EAClB,0BAA0B,EAC1B,cAAc,EACd,qBAAqB,EACrB,KAAK,YAAY,EACjB,KAAK,WAAW,EAChB,KAAK,mBAAmB,EACxB,KAAK,sBAAsB,EAC3B,KAAK,UAAU,EACf,KAAK,aAAa,EAClB,KAAK,YAAY,EACjB,KAAK,oBAAoB,EACzB,KAAK,QAAQ,EACb,KAAK,eAAe,GACrB,MAAM,oBAAoB,CAAC;AAC5B,OAAO,EACL,sBAAsB,EACtB,KAAK,gBAAgB,GACtB,MAAM,0BAA0B,CAAC;AAClC,OAAO,EACL,oBAAoB,EACpB,WAAW,EACX,mBAAmB,EACnB,gBAAgB,EAChB,KAAK,cAAc,EACnB,KAAK,KAAK,GACX,MAAM,6BAA6B,CAAC;AACrC,OAAO,EACL,iBAAiB,EACjB,SAAS,EACT,qBAAqB,EACrB,mBAAmB,EACnB,cAAc,EACd,6BAA6B,EAC7B,KAAK,WAAW,EAChB,KAAK,GAAG,EACR,KAAK,eAAe,EACpB,KAAK,aAAa,EAClB,KAAK,QAAQ,EACb,KAAK,uBAAuB,GAC7B,MAAM,0BAA0B,CAAC;AAClC,OAAO,EACL,sBAAsB,EACtB,oBAAoB,EACpB,oBAAoB,EACpB,sBAAsB,EACtB,uBAAuB,EACvB,2BAA2B,EAC3B,KAAK,gBAAgB,EACrB,KAAK,cAAc,EACnB,KAAK,cAAc,EACnB,KAAK,gBAAgB,EACrB,KAAK,iBAAiB,EACtB,KAAK,qBAAqB,GAC3B,MAAM,+BAA+B,CAAC;AACvC,OAAO,EACL,kBAAkB,EAClB,wBAAwB,EACxB,8BAA8B,EAC9B,kCAAkC,EAClC,2BAA2B,EAC3B,KAAK,YAAY,EACjB,KAAK,wBAAwB,EAC7B,KAAK,wBAAwB,GAC9B,MAAM,2BAA2B,CAAC;AACnC,OAAO,EACL,wBAAwB,EACxB,iCAAiC,EACjC,qCAAqC,EACrC,KAAK,kBAAkB,EACvB,KAAK,2BAA2B,EAChC,KAAK,+BAA+B,GACrC,MAAM,iCAAiC,CAAC;AACzC,OAAO,EACL,mBAAmB,EACnB,4BAA4B,EAC5B,6BAA6B,EAC7B,KAAK,aAAa,EAClB,KAAK,sBAAsB,EAC3B,KAAK,uBAAuB,GAC7B,MAAM,4BAA4B,CAAC;AACpC,OAAO,EACL,cAAc,EACd,kBAAkB,EAClB,KAAK,QAAQ,EACb,KAAK,YAAY,GAClB,MAAM,oBAAoB,CAAC;AAC5B,OAAO,EACL,wBAAwB,EACxB,kBAAkB,EAClB,uBAAuB,EACvB,qBAAqB,EACrB,sBAAsB,EACtB,uBAAuB,EACvB,4BAA4B,EAC5B,uBAAuB,EACvB,KAAK,kBAAkB,EACvB,KAAK,YAAY,EACjB,KAAK,iBAAiB,EACtB,KAAK,eAAe,EACpB,KAAK,gBAAgB,EACrB,KAAK,iBAAiB,EACtB,KAAK,sBAAsB,EAC3B,KAAK,iBAAiB,GACvB,MAAM,wBAAwB,CAAC;AAChC,OAAO,EACL,0BAA0B,EAC1B,mBAAmB,EACnB,eAAe,EACf,gBAAgB,EAChB,iBAAiB,EACjB,gBAAgB,EAChB,KAAK,oBAAoB,EACzB,KAAK,aAAa,EAClB,KAAK,SAAS,EACd,KAAK,UAAU,EACf,KAAK,WAAW,EAChB,KAAK,UAAU,GAChB,MAAM,mBAAmB,CAAC"}
|
package/dist/schemas/index.js
CHANGED
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
export { GoldenManifestSchema, GoldenSiteSchema, GoldenSiteExpectedSchema, } from './golden-manifest.schema.js';
|
|
1
2
|
export { HarnessConfigSchema, resolveMaxOutputTokensPerLlmCall, AuthConfigSchema, DetectedAuthSchema, AuthPathRequirementsSchema, AuthPathSchema, AuthExplorationSchema, } from './config.schema.js';
|
|
2
3
|
export { DecisionLogEntrySchema, } from './decision-log.schema.js';
|
|
3
4
|
export { RouteInventorySchema, RouteSchema, A11yViolationSchema, BrokenLinkSchema, } from './route-inventory.schema.js';
|
|
@@ -74,6 +74,8 @@ export declare const PublicSurfaceSchema: z.ZodObject<{
|
|
|
74
74
|
nodeCount: number;
|
|
75
75
|
}>, "many">;
|
|
76
76
|
statusCode: z.ZodOptional<z.ZodNumber>;
|
|
77
|
+
headers: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodString>>;
|
|
78
|
+
bodySnippet: z.ZodOptional<z.ZodString>;
|
|
77
79
|
}, "strip", z.ZodTypeAny, {
|
|
78
80
|
path: string;
|
|
79
81
|
pageTitle: string;
|
|
@@ -93,6 +95,8 @@ export declare const PublicSurfaceSchema: z.ZodObject<{
|
|
|
93
95
|
nodeCount: number;
|
|
94
96
|
}[];
|
|
95
97
|
statusCode?: number | undefined;
|
|
98
|
+
headers?: Record<string, string> | undefined;
|
|
99
|
+
bodySnippet?: string | undefined;
|
|
96
100
|
}, {
|
|
97
101
|
path: string;
|
|
98
102
|
pageTitle: string;
|
|
@@ -112,13 +116,15 @@ export declare const PublicSurfaceSchema: z.ZodObject<{
|
|
|
112
116
|
nodeCount: number;
|
|
113
117
|
}[];
|
|
114
118
|
statusCode?: number | undefined;
|
|
119
|
+
headers?: Record<string, string> | undefined;
|
|
120
|
+
bodySnippet?: string | undefined;
|
|
115
121
|
}>, "many">;
|
|
116
122
|
gaps: z.ZodArray<z.ZodObject<{
|
|
117
123
|
id: z.ZodString;
|
|
118
124
|
path: z.ZodString;
|
|
119
125
|
severity: z.ZodEnum<["critical", "high", "medium", "low"]>;
|
|
120
126
|
reason: z.ZodString;
|
|
121
|
-
category: z.ZodEnum<["untested-route", "a11y", "console-error", "broken-link", "auth-surface", "coverage", "untested-api-endpoint"]>;
|
|
127
|
+
category: z.ZodEnum<["untested-route", "a11y", "console-error", "broken-link", "auth-surface", "coverage", "untested-api-endpoint", "prompt-leakage"]>;
|
|
122
128
|
description: z.ZodOptional<z.ZodString>;
|
|
123
129
|
recommendation: z.ZodOptional<z.ZodString>;
|
|
124
130
|
}, "strip", z.ZodTypeAny, {
|
|
@@ -126,7 +132,7 @@ export declare const PublicSurfaceSchema: z.ZodObject<{
|
|
|
126
132
|
id: string;
|
|
127
133
|
severity: "critical" | "high" | "medium" | "low";
|
|
128
134
|
reason: string;
|
|
129
|
-
category: "untested-route" | "a11y" | "console-error" | "broken-link" | "auth-surface" | "coverage" | "untested-api-endpoint";
|
|
135
|
+
category: "untested-route" | "a11y" | "console-error" | "broken-link" | "auth-surface" | "coverage" | "untested-api-endpoint" | "prompt-leakage";
|
|
130
136
|
recommendation?: string | undefined;
|
|
131
137
|
description?: string | undefined;
|
|
132
138
|
}, {
|
|
@@ -134,7 +140,7 @@ export declare const PublicSurfaceSchema: z.ZodObject<{
|
|
|
134
140
|
id: string;
|
|
135
141
|
severity: "critical" | "high" | "medium" | "low";
|
|
136
142
|
reason: string;
|
|
137
|
-
category: "untested-route" | "a11y" | "console-error" | "broken-link" | "auth-surface" | "coverage" | "untested-api-endpoint";
|
|
143
|
+
category: "untested-route" | "a11y" | "console-error" | "broken-link" | "auth-surface" | "coverage" | "untested-api-endpoint" | "prompt-leakage";
|
|
138
144
|
recommendation?: string | undefined;
|
|
139
145
|
description?: string | undefined;
|
|
140
146
|
}>, "many">;
|
|
@@ -181,7 +187,7 @@ export declare const PublicSurfaceSchema: z.ZodObject<{
|
|
|
181
187
|
id: string;
|
|
182
188
|
severity: "critical" | "high" | "medium" | "low";
|
|
183
189
|
reason: string;
|
|
184
|
-
category: "untested-route" | "a11y" | "console-error" | "broken-link" | "auth-surface" | "coverage" | "untested-api-endpoint";
|
|
190
|
+
category: "untested-route" | "a11y" | "console-error" | "broken-link" | "auth-surface" | "coverage" | "untested-api-endpoint" | "prompt-leakage";
|
|
185
191
|
recommendation?: string | undefined;
|
|
186
192
|
description?: string | undefined;
|
|
187
193
|
}[];
|
|
@@ -210,6 +216,8 @@ export declare const PublicSurfaceSchema: z.ZodObject<{
|
|
|
210
216
|
nodeCount: number;
|
|
211
217
|
}[];
|
|
212
218
|
statusCode?: number | undefined;
|
|
219
|
+
headers?: Record<string, string> | undefined;
|
|
220
|
+
bodySnippet?: string | undefined;
|
|
213
221
|
}[];
|
|
214
222
|
accessibilityViolations: {
|
|
215
223
|
path: string;
|
|
@@ -224,7 +232,7 @@ export declare const PublicSurfaceSchema: z.ZodObject<{
|
|
|
224
232
|
id: string;
|
|
225
233
|
severity: "critical" | "high" | "medium" | "low";
|
|
226
234
|
reason: string;
|
|
227
|
-
category: "untested-route" | "a11y" | "console-error" | "broken-link" | "auth-surface" | "coverage" | "untested-api-endpoint";
|
|
235
|
+
category: "untested-route" | "a11y" | "console-error" | "broken-link" | "auth-surface" | "coverage" | "untested-api-endpoint" | "prompt-leakage";
|
|
228
236
|
recommendation?: string | undefined;
|
|
229
237
|
description?: string | undefined;
|
|
230
238
|
}[];
|
|
@@ -253,6 +261,8 @@ export declare const PublicSurfaceSchema: z.ZodObject<{
|
|
|
253
261
|
nodeCount: number;
|
|
254
262
|
}[];
|
|
255
263
|
statusCode?: number | undefined;
|
|
264
|
+
headers?: Record<string, string> | undefined;
|
|
265
|
+
bodySnippet?: string | undefined;
|
|
256
266
|
}[];
|
|
257
267
|
accessibilityViolations: {
|
|
258
268
|
path: string;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"public-surface.schema.d.ts","sourceRoot":"","sources":["../../src/schemas/public-surface.schema.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAIxB,eAAO,MAAM,4BAA4B;;;;;;;;;;;;;;;;;;;EAEvC,CAAC;AAEH,eAAO,MAAM,6BAA6B;;;;;;;;;;;;;;;;EAExC,CAAC;AAEH,eAAO,MAAM,mBAAmB
|
|
1
|
+
{"version":3,"file":"public-surface.schema.d.ts","sourceRoot":"","sources":["../../src/schemas/public-surface.schema.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAIxB,eAAO,MAAM,4BAA4B;;;;;;;;;;;;;;;;;;;EAEvC,CAAC;AAEH,eAAO,MAAM,6BAA6B;;;;;;;;;;;;;;;;EAExC,CAAC;AAEH,eAAO,MAAM,mBAAmB;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;EAK9B,CAAC;AAEH,MAAM,MAAM,aAAa,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,mBAAmB,CAAC,CAAC;AAChE,MAAM,MAAM,sBAAsB,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,4BAA4B,CAAC,CAAC;AAClF,MAAM,MAAM,uBAAuB,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,6BAA6B,CAAC,CAAC"}
|
|
@@ -65,6 +65,10 @@ export declare const RouteSchema: z.ZodObject<{
|
|
|
65
65
|
nodeCount: number;
|
|
66
66
|
}>, "many">;
|
|
67
67
|
statusCode: z.ZodOptional<z.ZodNumber>;
|
|
68
|
+
/** Optional: response headers from the page fetch (populated by explorers that capture them). */
|
|
69
|
+
headers: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodString>>;
|
|
70
|
+
/** Optional: first ~4000 chars of the raw HTML body (populated by explorers that capture it). */
|
|
71
|
+
bodySnippet: z.ZodOptional<z.ZodString>;
|
|
68
72
|
}, "strip", z.ZodTypeAny, {
|
|
69
73
|
path: string;
|
|
70
74
|
pageTitle: string;
|
|
@@ -84,6 +88,8 @@ export declare const RouteSchema: z.ZodObject<{
|
|
|
84
88
|
nodeCount: number;
|
|
85
89
|
}[];
|
|
86
90
|
statusCode?: number | undefined;
|
|
91
|
+
headers?: Record<string, string> | undefined;
|
|
92
|
+
bodySnippet?: string | undefined;
|
|
87
93
|
}, {
|
|
88
94
|
path: string;
|
|
89
95
|
pageTitle: string;
|
|
@@ -103,6 +109,8 @@ export declare const RouteSchema: z.ZodObject<{
|
|
|
103
109
|
nodeCount: number;
|
|
104
110
|
}[];
|
|
105
111
|
statusCode?: number | undefined;
|
|
112
|
+
headers?: Record<string, string> | undefined;
|
|
113
|
+
bodySnippet?: string | undefined;
|
|
106
114
|
}>;
|
|
107
115
|
export declare const RouteInventorySchema: z.ZodObject<{
|
|
108
116
|
scannedAt: z.ZodString;
|
|
@@ -144,6 +152,10 @@ export declare const RouteInventorySchema: z.ZodObject<{
|
|
|
144
152
|
nodeCount: number;
|
|
145
153
|
}>, "many">;
|
|
146
154
|
statusCode: z.ZodOptional<z.ZodNumber>;
|
|
155
|
+
/** Optional: response headers from the page fetch (populated by explorers that capture them). */
|
|
156
|
+
headers: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodString>>;
|
|
157
|
+
/** Optional: first ~4000 chars of the raw HTML body (populated by explorers that capture it). */
|
|
158
|
+
bodySnippet: z.ZodOptional<z.ZodString>;
|
|
147
159
|
}, "strip", z.ZodTypeAny, {
|
|
148
160
|
path: string;
|
|
149
161
|
pageTitle: string;
|
|
@@ -163,6 +175,8 @@ export declare const RouteInventorySchema: z.ZodObject<{
|
|
|
163
175
|
nodeCount: number;
|
|
164
176
|
}[];
|
|
165
177
|
statusCode?: number | undefined;
|
|
178
|
+
headers?: Record<string, string> | undefined;
|
|
179
|
+
bodySnippet?: string | undefined;
|
|
166
180
|
}, {
|
|
167
181
|
path: string;
|
|
168
182
|
pageTitle: string;
|
|
@@ -182,6 +196,8 @@ export declare const RouteInventorySchema: z.ZodObject<{
|
|
|
182
196
|
nodeCount: number;
|
|
183
197
|
}[];
|
|
184
198
|
statusCode?: number | undefined;
|
|
199
|
+
headers?: Record<string, string> | undefined;
|
|
200
|
+
bodySnippet?: string | undefined;
|
|
185
201
|
}>, "many">;
|
|
186
202
|
pagesSkipped: z.ZodNumber;
|
|
187
203
|
budgetExceeded: z.ZodBoolean;
|
|
@@ -207,6 +223,8 @@ export declare const RouteInventorySchema: z.ZodObject<{
|
|
|
207
223
|
nodeCount: number;
|
|
208
224
|
}[];
|
|
209
225
|
statusCode?: number | undefined;
|
|
226
|
+
headers?: Record<string, string> | undefined;
|
|
227
|
+
bodySnippet?: string | undefined;
|
|
210
228
|
}[];
|
|
211
229
|
pagesSkipped: number;
|
|
212
230
|
budgetExceeded: boolean;
|
|
@@ -232,6 +250,8 @@ export declare const RouteInventorySchema: z.ZodObject<{
|
|
|
232
250
|
nodeCount: number;
|
|
233
251
|
}[];
|
|
234
252
|
statusCode?: number | undefined;
|
|
253
|
+
headers?: Record<string, string> | undefined;
|
|
254
|
+
bodySnippet?: string | undefined;
|
|
235
255
|
}[];
|
|
236
256
|
pagesSkipped: number;
|
|
237
257
|
budgetExceeded: boolean;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"route-inventory.schema.d.ts","sourceRoot":"","sources":["../../src/schemas/route-inventory.schema.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAExB,eAAO,MAAM,mBAAmB;;;;;;;;;;;;;;;EAK9B,CAAC;AAEH,eAAO,MAAM,gBAAgB;;;;;;;;;;;;EAI3B,CAAC;AAEH,eAAO,MAAM,WAAW
|
|
1
|
+
{"version":3,"file":"route-inventory.schema.d.ts","sourceRoot":"","sources":["../../src/schemas/route-inventory.schema.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAExB,eAAO,MAAM,mBAAmB;;;;;;;;;;;;;;;EAK9B,CAAC;AAEH,eAAO,MAAM,gBAAgB;;;;;;;;;;;;EAI3B,CAAC;AAEH,eAAO,MAAM,WAAW;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;IAUtB,iGAAiG;;IAEjG,iGAAiG;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;EAEjG,CAAC;AAEH,eAAO,MAAM,oBAAoB;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;QAN/B,iGAAiG;;QAEjG,iGAAiG;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;EAUjG,CAAC;AAEH,MAAM,MAAM,cAAc,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,oBAAoB,CAAC,CAAC;AAClE,MAAM,MAAM,KAAK,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,WAAW,CAAC,CAAC"}
|
|
@@ -20,6 +20,10 @@ export const RouteSchema = z.object({
|
|
|
20
20
|
brokenLinks: z.array(BrokenLinkSchema),
|
|
21
21
|
a11yViolations: z.array(A11yViolationSchema),
|
|
22
22
|
statusCode: z.number().int().optional(),
|
|
23
|
+
/** Optional: response headers from the page fetch (populated by explorers that capture them). */
|
|
24
|
+
headers: z.record(z.string(), z.string()).optional(),
|
|
25
|
+
/** Optional: first ~4000 chars of the raw HTML body (populated by explorers that capture it). */
|
|
26
|
+
bodySnippet: z.string().max(8000).optional(),
|
|
23
27
|
});
|
|
24
28
|
export const RouteInventorySchema = z.object({
|
|
25
29
|
scannedAt: z.string().datetime(),
|
|
@@ -211,8 +211,8 @@ export declare const AuditEntrySchema: z.ZodObject<{
|
|
|
211
211
|
recordHash: z.ZodString;
|
|
212
212
|
}, "strip", z.ZodTypeAny, {
|
|
213
213
|
computedAt: string;
|
|
214
|
-
tenantId: string;
|
|
215
214
|
schemaVersion: 1;
|
|
215
|
+
tenantId: string;
|
|
216
216
|
confidenceScore: number | null;
|
|
217
217
|
verdict: "ship" | "caution" | "hold" | "block";
|
|
218
218
|
blockers: string[];
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"confidence.d.ts","sourceRoot":"","sources":["../../../src/tools/scoring/confidence.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;GAuBG;AAEH,OAAO,KAAK,EACV,eAAe,
|
|
1
|
+
{"version":3,"file":"confidence.d.ts","sourceRoot":"","sources":["../../../src/tools/scoring/confidence.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;GAuBG;AAEH,OAAO,KAAK,EACV,eAAe,EAIf,iBAAiB,EAElB,MAAM,oCAAoC,CAAC;AA8L5C;;;;GAIG;AACH,wBAAgB,wBAAwB,CAAC,KAAK,EAAE,eAAe,GAAG,iBAAiB,CAgKlF"}
|
|
@@ -43,6 +43,18 @@ const DEFAULT_WEIGHTS = {
|
|
|
43
43
|
'human-approval': 0.0,
|
|
44
44
|
'agent-evidence': 0.0,
|
|
45
45
|
};
|
|
46
|
+
/** Model sources with non-zero default weight — the full evidence model for partial-run disclosure. */
|
|
47
|
+
const MODEL_SOURCES = Object.entries(DEFAULT_WEIGHTS)
|
|
48
|
+
.filter(([, weight]) => weight > 0)
|
|
49
|
+
.map(([source]) => source);
|
|
50
|
+
const UNCOLLECTED_NEXT_CHECKS = {
|
|
51
|
+
'live-app-quality': 'Run analyze_app against the deployed URL to collect live-app quality evidence.',
|
|
52
|
+
'accessibility': 'Run analyze_app against the deployed URL to evaluate accessibility.',
|
|
53
|
+
'crawl-coverage': 'Run analyze_app against the deployed URL to measure crawl coverage.',
|
|
54
|
+
'test-automation': 'Run qulib score-automation against the repo to score test automation maturity.',
|
|
55
|
+
'api-coverage': 'Run qulib score-api against the repo to measure API test coverage.',
|
|
56
|
+
'ci-results': 'Ingest CI status from your pipeline (ci-results source not yet wired).',
|
|
57
|
+
};
|
|
46
58
|
function resolvePolicy(p) {
|
|
47
59
|
const base = ConfidencePolicySchema.parse(p ?? {});
|
|
48
60
|
return {
|
|
@@ -72,6 +84,93 @@ function buildHonestyNote(item) {
|
|
|
72
84
|
}
|
|
73
85
|
return `${base} has partial or degraded signal.`;
|
|
74
86
|
}
|
|
87
|
+
function resolveModelWeight(source, policyWeights) {
|
|
88
|
+
if (policyWeights && source in policyWeights) {
|
|
89
|
+
return policyWeights[source];
|
|
90
|
+
}
|
|
91
|
+
return DEFAULT_WEIGHTS[source] ?? 0;
|
|
92
|
+
}
|
|
93
|
+
function inferUncollectedReason(source, presentSources) {
|
|
94
|
+
const hasAnalyzeEvidence = presentSources.has('live-app-quality') ||
|
|
95
|
+
presentSources.has('accessibility') ||
|
|
96
|
+
presentSources.has('crawl-coverage');
|
|
97
|
+
const hasRepoEvidence = presentSources.has('test-automation') || presentSources.has('api-coverage');
|
|
98
|
+
switch (source) {
|
|
99
|
+
case 'live-app-quality':
|
|
100
|
+
case 'accessibility':
|
|
101
|
+
case 'crawl-coverage':
|
|
102
|
+
return hasAnalyzeEvidence
|
|
103
|
+
? 'not collected in this confidence run'
|
|
104
|
+
: 'app-runtime analysis not run — no url provided';
|
|
105
|
+
case 'test-automation':
|
|
106
|
+
case 'api-coverage':
|
|
107
|
+
return hasRepoEvidence
|
|
108
|
+
? 'not collected in this confidence run'
|
|
109
|
+
: 'repo scoring not run — no repo provided';
|
|
110
|
+
case 'ci-results':
|
|
111
|
+
return 'CI status not ingested — no ci-results source wired';
|
|
112
|
+
default:
|
|
113
|
+
return 'not collected';
|
|
114
|
+
}
|
|
115
|
+
}
|
|
116
|
+
function buildUncollectedHonestyNote(source, reason, rawWeight) {
|
|
117
|
+
const pct = Math.round(rawWeight * 100);
|
|
118
|
+
return `'${source}' not collected (${pct}% raw model weight): ${reason}.`;
|
|
119
|
+
}
|
|
120
|
+
function buildCoverageSummaryNote(scoredSourceCount, modelSourceCount, rawWeightScored, rawWeightModel) {
|
|
121
|
+
const coveragePct = rawWeightModel > 0 ? Math.round((rawWeightScored / rawWeightModel) * 100) : 0;
|
|
122
|
+
return (`Partial evidence: verdict computed on ${scoredSourceCount} of ${modelSourceCount} model sources ` +
|
|
123
|
+
`(~${coveragePct}% of raw model weight). Collected weights were renormalized to 100% for the score.`);
|
|
124
|
+
}
|
|
125
|
+
function isPositiveEvidence(text) {
|
|
126
|
+
if (/appear covered/i.test(text))
|
|
127
|
+
return true;
|
|
128
|
+
if (/Automation maturity: L\d/i.test(text))
|
|
129
|
+
return true;
|
|
130
|
+
if (/No a11y gaps/i.test(text))
|
|
131
|
+
return true;
|
|
132
|
+
if (/^L\d —/i.test(text))
|
|
133
|
+
return true;
|
|
134
|
+
if (/^releaseConfidence=/i.test(text))
|
|
135
|
+
return true;
|
|
136
|
+
if (/^coverageScore=/i.test(text))
|
|
137
|
+
return true;
|
|
138
|
+
if (/^No .* gaps detected/i.test(text))
|
|
139
|
+
return true;
|
|
140
|
+
return false;
|
|
141
|
+
}
|
|
142
|
+
function extractItemRisks(item, passThreshold) {
|
|
143
|
+
const risks = [];
|
|
144
|
+
if (item.blocking) {
|
|
145
|
+
if (item.reason)
|
|
146
|
+
risks.push(item.reason);
|
|
147
|
+
risks.push(...item.evidence.filter((entry) => !isPositiveEvidence(entry)));
|
|
148
|
+
return risks;
|
|
149
|
+
}
|
|
150
|
+
const applicability = item.applicability ?? 'applicable';
|
|
151
|
+
if (applicability === 'unknown' || item.score === null) {
|
|
152
|
+
if (item.reason)
|
|
153
|
+
risks.push(`${item.source}: ${item.reason}`);
|
|
154
|
+
risks.push(...item.evidence.filter((entry) => !isPositiveEvidence(entry) && /(gap|critical|high|untested|uncovered|missing|block|fail|warning|auth|blocked)/i.test(entry)));
|
|
155
|
+
return risks;
|
|
156
|
+
}
|
|
157
|
+
if (applicability === 'not_applicable') {
|
|
158
|
+
if (item.reason)
|
|
159
|
+
risks.push(`${item.source}: ${item.reason}`);
|
|
160
|
+
return risks;
|
|
161
|
+
}
|
|
162
|
+
if (item.score !== null && item.score < passThreshold) {
|
|
163
|
+
risks.push(...item.evidence.filter((entry) => !isPositiveEvidence(entry)));
|
|
164
|
+
if (item.score < passThreshold) {
|
|
165
|
+
risks.push(`${item.source} scored ${item.score}/100 — below pass threshold (${passThreshold}).`);
|
|
166
|
+
}
|
|
167
|
+
}
|
|
168
|
+
else {
|
|
169
|
+
risks.push(...item.evidence.filter((entry) => !isPositiveEvidence(entry) &&
|
|
170
|
+
/(gap|critical|high|untested|uncovered|missing|block|fail|warning|penalty|below)/i.test(entry)));
|
|
171
|
+
}
|
|
172
|
+
return risks;
|
|
173
|
+
}
|
|
75
174
|
/**
|
|
76
175
|
* Compute the fused Release Confidence result from an evidence bundle.
|
|
77
176
|
*
|
|
@@ -137,29 +236,56 @@ export function computeReleaseConfidence(input) {
|
|
|
137
236
|
}
|
|
138
237
|
// Level / label from shared ladder.
|
|
139
238
|
const { level, label } = scoreLevel(confidenceScore ?? 0);
|
|
140
|
-
|
|
239
|
+
const presentSources = new Set(input.evidence.map((item) => item.source));
|
|
240
|
+
const uncollectedSources = MODEL_SOURCES.filter((source) => !presentSources.has(source));
|
|
241
|
+
const modelWeightSum = MODEL_SOURCES.reduce((sum, source) => sum + resolveModelWeight(source, policy.weights), 0);
|
|
242
|
+
// Honesty notes — partial-run summary first, then present-but-excluded sources (must not
|
|
243
|
+
// be truncated by maxListLength), then uncollected model sources.
|
|
141
244
|
const honestyNotes = [];
|
|
245
|
+
if (uncollectedSources.length > 0 || (weightSum > 0 && weightSum < modelWeightSum - 0.001)) {
|
|
246
|
+
honestyNotes.push(buildCoverageSummaryNote(applicable.length, MODEL_SOURCES.length, weightSum, modelWeightSum));
|
|
247
|
+
}
|
|
142
248
|
for (const item of excluded) {
|
|
143
249
|
honestyNotes.push(buildHonestyNote(item));
|
|
144
250
|
}
|
|
145
|
-
|
|
251
|
+
for (const source of uncollectedSources) {
|
|
252
|
+
const rawWeight = resolveModelWeight(source, policy.weights);
|
|
253
|
+
const reason = inferUncollectedReason(source, presentSources);
|
|
254
|
+
honestyNotes.push(buildUncollectedHonestyNote(source, reason, rawWeight));
|
|
255
|
+
}
|
|
146
256
|
for (const item of blockingItems) {
|
|
147
257
|
if ((item.applicability ?? 'applicable') === 'applicable' && item.score !== null) {
|
|
148
258
|
honestyNotes.push(`'${item.source}' is a hard blocker${item.reason ? ': ' + item.reason : ''}.`);
|
|
149
259
|
}
|
|
150
260
|
}
|
|
151
|
-
// Top risks —
|
|
152
|
-
const allRisks = [
|
|
153
|
-
|
|
154
|
-
|
|
155
|
-
|
|
156
|
-
|
|
157
|
-
.
|
|
158
|
-
|
|
159
|
-
|
|
160
|
-
|
|
161
|
-
|
|
162
|
-
|
|
261
|
+
// Top risks — gaps and blockers only; never surface coverage successes as risks.
|
|
262
|
+
const allRisks = [];
|
|
263
|
+
for (const source of uncollectedSources) {
|
|
264
|
+
const rawWeight = resolveModelWeight(source, policy.weights);
|
|
265
|
+
if (rawWeight >= 0.10) {
|
|
266
|
+
const reason = inferUncollectedReason(source, presentSources);
|
|
267
|
+
allRisks.push(`Uncollected high-weight evidence: ${source} (${Math.round(rawWeight * 100)}% raw weight) — ${reason}.`);
|
|
268
|
+
}
|
|
269
|
+
}
|
|
270
|
+
for (const item of blockingItems) {
|
|
271
|
+
allRisks.push(...extractItemRisks(item, policy.passThreshold));
|
|
272
|
+
}
|
|
273
|
+
for (const item of [...excluded].sort((a, b) => resolveWeight(a, policy.weights) - resolveWeight(b, policy.weights))) {
|
|
274
|
+
allRisks.push(...extractItemRisks(item, policy.passThreshold));
|
|
275
|
+
}
|
|
276
|
+
for (const item of [...applicable].sort((a, b) => (a.score ?? 0) - (b.score ?? 0))) {
|
|
277
|
+
allRisks.push(...extractItemRisks(item, policy.passThreshold));
|
|
278
|
+
}
|
|
279
|
+
const topRisks = [...new Set(allRisks.filter(Boolean))].slice(0, limit);
|
|
280
|
+
// Recommended next checks — concrete actions for uncollected sources plus per-item recommendations.
|
|
281
|
+
const allRecs = [];
|
|
282
|
+
for (const source of uncollectedSources) {
|
|
283
|
+
const rec = UNCOLLECTED_NEXT_CHECKS[source];
|
|
284
|
+
if (rec)
|
|
285
|
+
allRecs.push(rec);
|
|
286
|
+
}
|
|
287
|
+
allRecs.push(...input.evidence.flatMap((item) => item.recommendations ?? []));
|
|
288
|
+
const recommendedNextChecks = [...new Set(allRecs.filter(Boolean))].slice(0, limit);
|
|
163
289
|
const result = {
|
|
164
290
|
schemaVersion: 1,
|
|
165
291
|
computedAt: now,
|
|
@@ -0,0 +1,29 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Prompt-leakage detector — gap category `prompt-leakage`.
|
|
3
|
+
*
|
|
4
|
+
* Flags when a web page inadvertently exposes AI system-prompt / agent
|
|
5
|
+
* instructions in its public surface: inline scripts, HTML comments, meta
|
|
6
|
+
* tags, visible text, response headers, or error bodies.
|
|
7
|
+
*
|
|
8
|
+
* CONSERVATIVE design: every signal requires TWO corroborating markers
|
|
9
|
+
* before generating a Gap, to keep the false-positive rate low.
|
|
10
|
+
* A page that merely uses the word "AI" or "assistant" will NOT trip.
|
|
11
|
+
*
|
|
12
|
+
* Heuristics are derived from first principles — the structural telltale
|
|
13
|
+
* shapes of an exposed instruction block. No third-party leaked-prompt
|
|
14
|
+
* text or vendor identifiers were used.
|
|
15
|
+
*/
|
|
16
|
+
import type { Gap } from '../../schemas/gap-analysis.schema.js';
|
|
17
|
+
import type { Route } from '../../schemas/route-inventory.schema.js';
|
|
18
|
+
/**
|
|
19
|
+
* Scan a captured page surface for signals that an AI system prompt or agent
|
|
20
|
+
* instructions are exposed in its public surface.
|
|
21
|
+
*
|
|
22
|
+
* Accepts the `Route` shape from `route-inventory.schema.ts`, which now
|
|
23
|
+
* includes the optional `headers` and `bodySnippet` fields.
|
|
24
|
+
*
|
|
25
|
+
* Returns an array of `Gap` objects with `category: 'prompt-leakage'`.
|
|
26
|
+
* Returns an empty array when no signals are found.
|
|
27
|
+
*/
|
|
28
|
+
export declare function detectPromptLeakage(route: Pick<Route, 'path' | 'headers' | 'bodySnippet'>): Gap[];
|
|
29
|
+
//# sourceMappingURL=prompt-leakage.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"prompt-leakage.d.ts","sourceRoot":"","sources":["../../../src/tools/scoring/prompt-leakage.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;GAcG;AAGH,OAAO,KAAK,EAAE,GAAG,EAAE,MAAM,sCAAsC,CAAC;AAChE,OAAO,KAAK,EAAE,KAAK,EAAE,MAAM,yCAAyC,CAAC;AAqLrE;;;;;;;;;GASG;AACH,wBAAgB,mBAAmB,CAAC,KAAK,EAAE,IAAI,CAAC,KAAK,EAAE,MAAM,GAAG,SAAS,GAAG,aAAa,CAAC,GAAG,GAAG,EAAE,CAgGjG"}
|