@contextrail/assess 0.0.0-stage → 1.0.0-alpha.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (73) hide show
  1. package/LICENSE +202 -0
  2. package/README.md +82 -2
  3. package/assets/brand/icon-dark.png +0 -0
  4. package/assets/brand/icon-light.png +0 -0
  5. package/dist/assess.d.ts +109 -0
  6. package/dist/assess.js +1548 -0
  7. package/dist/assessment.d.ts +566 -0
  8. package/dist/assessment.js +404 -0
  9. package/dist/budget.d.ts +24 -0
  10. package/dist/budget.js +52 -0
  11. package/dist/candidate.d.ts +147 -0
  12. package/dist/candidate.js +321 -0
  13. package/dist/change-summary.d.ts +33 -0
  14. package/dist/change-summary.js +26 -0
  15. package/dist/cli.d.ts +2 -0
  16. package/dist/cli.js +406 -0
  17. package/dist/contracts.d.ts +99 -0
  18. package/dist/contracts.js +62 -0
  19. package/dist/diagnostics.d.ts +44 -0
  20. package/dist/diagnostics.js +412 -0
  21. package/dist/evidence.d.ts +73 -0
  22. package/dist/evidence.js +297 -0
  23. package/dist/feedback.d.ts +21 -0
  24. package/dist/feedback.js +140 -0
  25. package/dist/git.d.ts +40 -0
  26. package/dist/git.js +207 -0
  27. package/dist/github.d.ts +90 -0
  28. package/dist/github.js +1342 -0
  29. package/dist/index.d.ts +14 -0
  30. package/dist/index.js +14 -0
  31. package/dist/mcp.d.ts +28 -0
  32. package/dist/mcp.js +173 -0
  33. package/dist/pack.d.ts +160 -0
  34. package/dist/pack.js +377 -0
  35. package/dist/phases.d.ts +65 -0
  36. package/dist/phases.js +63 -0
  37. package/dist/planning.d.ts +64 -0
  38. package/dist/planning.js +135 -0
  39. package/dist/pr-context.d.ts +316 -0
  40. package/dist/pr-context.js +198 -0
  41. package/dist/prepare.d.ts +71 -0
  42. package/dist/prepare.js +335 -0
  43. package/dist/presentation.d.ts +309 -0
  44. package/dist/presentation.js +1026 -0
  45. package/dist/provider-diagnostics.d.ts +93 -0
  46. package/dist/provider-diagnostics.js +233 -0
  47. package/dist/providers.d.ts +45 -0
  48. package/dist/providers.js +291 -0
  49. package/dist/report-contract.d.ts +132 -0
  50. package/dist/report-contract.js +240 -0
  51. package/dist/results.d.ts +90 -0
  52. package/dist/results.js +128 -0
  53. package/dist/resume.d.ts +25 -0
  54. package/dist/resume.js +116 -0
  55. package/dist/source.d.ts +167 -0
  56. package/dist/source.js +675 -0
  57. package/dist/verification.d.ts +386 -0
  58. package/dist/verification.js +168 -0
  59. package/dist/version.d.ts +2 -0
  60. package/dist/version.js +9 -0
  61. package/docs/README.md +24 -0
  62. package/docs/action-reference.md +85 -0
  63. package/docs/architecture.md +156 -0
  64. package/docs/configuration.md +83 -0
  65. package/docs/contributing.md +46 -0
  66. package/docs/getting-started.md +92 -0
  67. package/docs/github-actions.md +141 -0
  68. package/docs/operations.md +203 -0
  69. package/docs/review-results.md +173 -0
  70. package/docs/security.md +64 -0
  71. package/docs/standards.md +62 -0
  72. package/examples/assess-pull-request.yml +109 -0
  73. package/package.json +56 -5
@@ -0,0 +1,412 @@
1
+ import { appendFile } from 'node:fs/promises';
2
+ import { join } from 'node:path';
3
+ import { CitationError } from './evidence.js';
4
+ const fields = new Set([
5
+ 'phase',
6
+ 'workUnitId',
7
+ 'attempt',
8
+ 'call',
9
+ 'status',
10
+ 'code',
11
+ 'tool',
12
+ 'count',
13
+ 'bytes',
14
+ 'readCount',
15
+ 'elapsedMs',
16
+ 'hash',
17
+ 'inputHash',
18
+ 'outputHash',
19
+ 'evidenceRefs',
20
+ 'stopReason',
21
+ 'provider',
22
+ 'model',
23
+ 'usage',
24
+ 'inputTokens',
25
+ 'outputTokens',
26
+ 'totalTokens',
27
+ 'cachedInputTokens',
28
+ 'reasoningTokens',
29
+ 'cost',
30
+ 'missing',
31
+ 'details',
32
+ 'causes',
33
+ 'parse',
34
+ 'retryScope',
35
+ 'retryable',
36
+ 'criteriaPreserved',
37
+ 'backoffMs',
38
+ 'validation',
39
+ 'citation',
40
+ 'ref',
41
+ 'refHash',
42
+ 'path',
43
+ 'expected',
44
+ 'actual',
45
+ 'finishReason',
46
+ 'reporting',
47
+ 'exitCode',
48
+ 'complete',
49
+ 'knownCost',
50
+ 'usageComplete',
51
+ 'corrections',
52
+ 'assessmentId',
53
+ 'entryIndex',
54
+ 'checkIndex',
55
+ 'rejectedRefHash',
56
+ 'allowedRefCount',
57
+ 'omittedCount',
58
+ 'newlyAccepted',
59
+ 'alreadyAccepted',
60
+ 'remaining',
61
+ 'request',
62
+ 'requestBytes',
63
+ 'messageCount',
64
+ 'toolCount',
65
+ 'toolSchemaBytes',
66
+ 'maxOutputTokens',
67
+ 'requestFormat',
68
+ 'reason',
69
+ 'parameter',
70
+ 'evidence',
71
+ 'bodyFormat',
72
+ 'transport',
73
+ 'stage',
74
+ 'primaryStopReason',
75
+ 'systemCode',
76
+ ]);
77
+ const secretValues = () => Object.entries(process.env)
78
+ .filter(([key, value]) => /KEY|TOKEN|SECRET|PASSWORD/u.test(key) && value && value.length >= 8)
79
+ .map(([, value]) => value);
80
+ /** Operational records may retain assessment text; diagnostics never receive those bodies. */
81
+ export function sanitizeRecord(value) {
82
+ const secrets = secretValues().sort((left, right) => right.length - left.length);
83
+ let serialized = JSON.stringify(value, (_key, item) => {
84
+ if (typeof item !== 'string')
85
+ return item;
86
+ let text = item;
87
+ for (const secret of secrets)
88
+ text = text.split(secret).join('[REDACTED]');
89
+ return text;
90
+ });
91
+ if (serialized === undefined)
92
+ return value;
93
+ // Keys can also contain escaped values; operate on their JSON encoding, never raw escape fragments.
94
+ for (const secret of secrets)
95
+ serialized = serialized.split(JSON.stringify(secret).slice(1, -1)).join('[REDACTED]');
96
+ return JSON.parse(serialized);
97
+ }
98
+ function bounded(value, depth = 0, arrayLimit = 32) {
99
+ if (depth > 6)
100
+ return null;
101
+ if (value === null || typeof value === 'boolean')
102
+ return value;
103
+ if (typeof value === 'number')
104
+ return Number.isFinite(value) ? value : null;
105
+ if (typeof value === 'string')
106
+ return /^[A-Za-z0-9_./:@+[\] -]{0,220}$/u.test(value) ? value : '[omitted]';
107
+ if (Array.isArray(value))
108
+ return value.slice(0, arrayLimit).map((v) => bounded(v, depth + 1));
109
+ if (value && typeof value === 'object')
110
+ return Object.fromEntries(Object.entries(value)
111
+ .filter(([key]) => fields.has(key))
112
+ .map(([key, item]) => [
113
+ key,
114
+ key === 'citation'
115
+ ? safeCitationDetails(item)
116
+ : key === 'ref'
117
+ ? safeEvidenceRef(item)
118
+ : key === 'refHash'
119
+ ? safeRefHash(item)
120
+ : bounded(item, depth + 1, key === 'corrections' ? 100 : 32),
121
+ ]));
122
+ return null;
123
+ }
124
+ /** Fixed operating-system categories; no paths or exception messages leave the process. */
125
+ export function systemFailureDiagnostics(error) {
126
+ const codes = {
127
+ EACCES: 'filesystem_permission_denied',
128
+ EPERM: 'filesystem_permission_denied',
129
+ ENOSPC: 'filesystem_storage_full',
130
+ EDQUOT: 'filesystem_quota_exceeded',
131
+ ENOENT: 'filesystem_path_missing',
132
+ ENOTDIR: 'filesystem_path_invalid',
133
+ EISDIR: 'filesystem_path_invalid',
134
+ EROFS: 'filesystem_read_only',
135
+ EMFILE: 'filesystem_open_limit',
136
+ ENFILE: 'filesystem_open_limit',
137
+ };
138
+ let current = error;
139
+ for (let depth = 0; current && typeof current === 'object' && depth < 5; depth++) {
140
+ const code = ownData(current, 'code');
141
+ if (typeof code === 'string' && Object.hasOwn(codes, code))
142
+ return { systemCode: codes[code] };
143
+ current = ownData(current, 'cause');
144
+ }
145
+ return { systemCode: null };
146
+ }
147
+ export class Diagnostics {
148
+ directory;
149
+ runId;
150
+ level;
151
+ log;
152
+ writes = Promise.resolve();
153
+ writeFailed = false;
154
+ constructor(directory, runId, level, log = (line) => process.stderr.write(line)) {
155
+ this.directory = directory;
156
+ this.runId = runId;
157
+ this.level = level;
158
+ this.log = log;
159
+ }
160
+ event(kind, data = {}) {
161
+ const record = sanitizeRecord({ time: new Date().toISOString(), runId: this.runId, kind, data: bounded(data) });
162
+ const operation = this.writes.then(() => appendFile(join(this.directory, 'diagnostics.jsonl'), JSON.stringify(record) + '\n', { mode: 0o600 }));
163
+ this.writes = operation.catch(() => {
164
+ this.writeFailed = true;
165
+ });
166
+ if (kind.endsWith('_failed') ||
167
+ kind === 'execution_stopped' ||
168
+ this.level === 'debug' ||
169
+ (this.level === 'info' &&
170
+ [
171
+ 'phase_started',
172
+ 'phase_completed',
173
+ 'phase_failed',
174
+ 'provider_started',
175
+ 'provider_completed',
176
+ 'assessment_progress',
177
+ 'verification_progress',
178
+ 'assessment_continuation',
179
+ 'verification_continuation',
180
+ 'assessment_reporting',
181
+ 'provider_retry',
182
+ 'provider_recovered',
183
+ 'provider_retry_exhausted',
184
+ 'assessment_finalized',
185
+ ].includes(kind)))
186
+ try {
187
+ this.log(JSON.stringify(record) + '\n');
188
+ }
189
+ catch {
190
+ this.writeFailed = true;
191
+ }
192
+ return operation;
193
+ }
194
+ async flush() {
195
+ await this.writes;
196
+ if (this.writeFailed)
197
+ throw new Error('diagnostic_write_failed');
198
+ }
199
+ }
200
+ const validatorCodes = new Set([
201
+ 'assessment_coverage_mismatch',
202
+ 'check_requires_own_governing_passage',
203
+ 'structured_requirement_unaccounted',
204
+ 'check_source_evidence_required',
205
+ 'missing_evidence_need_required',
206
+ 'unresolved_required_check',
207
+ 'nonapplicability_check_mismatch',
208
+ 'distinct_finding_required',
209
+ 'finding_aggregate_mismatch',
210
+ 'blocking_finding_aggregate_mismatch',
211
+ 'foreign_finding_check',
212
+ 'foreign_assessment_id',
213
+ 'finding_source_required',
214
+ 'invalid_replacement_range',
215
+ 'verification_governing_evidence_required',
216
+ 'verification_acceptance_inconsistent',
217
+ 'verification_source_required',
218
+ 'verification_amendment_inconsistent',
219
+ 'verification_amendment_suggestion_unchecked',
220
+ 'verification_resolution_inconsistent',
221
+ 'verification_resolution_missing_checks',
222
+ 'verification_resolution_findings_mismatch',
223
+ 'unproposed_suggestion_acceptance',
224
+ 'phase_already_submitted',
225
+ ]);
226
+ const schemaFields = new Set([
227
+ 'entries',
228
+ 'assessmentId',
229
+ 'applicability',
230
+ 'reason',
231
+ 'workUnit',
232
+ 'checks',
233
+ 'passage',
234
+ 'interpretation',
235
+ 'surfaces',
236
+ 'evidenceNeeds',
237
+ 'assessments',
238
+ 'result',
239
+ 'revisionOf',
240
+ 'disposition',
241
+ 'adherence',
242
+ 'confidence',
243
+ 'confidenceReason',
244
+ 'finding',
245
+ 'explanation',
246
+ 'consequence',
247
+ 'correction',
248
+ 'limitations',
249
+ 'citations',
250
+ 'ref',
251
+ 'quote',
252
+ 'start',
253
+ 'end',
254
+ 'startSegment',
255
+ 'endSegment',
256
+ 'checkId',
257
+ 'sourceRefs',
258
+ 'contraryEvidence',
259
+ 'missingEvidence',
260
+ 'findings',
261
+ 'classification',
262
+ 'checkIds',
263
+ 'locationRefs',
264
+ 'proposedReplacement',
265
+ 'path',
266
+ 'blob',
267
+ 'startLine',
268
+ 'endLine',
269
+ 'before',
270
+ 'replacement',
271
+ 'missingChecks',
272
+ 'resolution',
273
+ 'amendment',
274
+ 'findingId',
275
+ 'accepted',
276
+ 'suggestionAccepted',
277
+ ]);
278
+ const schemaCodes = new Set([
279
+ 'invalid_type',
280
+ 'invalid_value',
281
+ 'too_small',
282
+ 'too_big',
283
+ 'invalid_format',
284
+ 'unrecognized_keys',
285
+ 'invalid_union',
286
+ ]);
287
+ const citationCodes = new Set(['citation_unknown', 'citation_absent', 'citation_ambiguous', 'citation_range_invalid']);
288
+ const safeEvidenceRef = (value) => typeof value === 'string' && /^evidence:[a-f0-9]{64}$/u.test(value) ? value : null;
289
+ const safeRefHash = (value) => typeof value === 'string' && /^sha256:[a-f0-9]{64}$/u.test(value) ? value : null;
290
+ function ownData(value, key) {
291
+ if (!value || typeof value !== 'object')
292
+ return undefined;
293
+ try {
294
+ return Object.getOwnPropertyDescriptor(value, key)?.value;
295
+ }
296
+ catch {
297
+ return undefined;
298
+ }
299
+ }
300
+ function citationPath(value) {
301
+ if (!Array.isArray(value))
302
+ return [];
303
+ try {
304
+ return Array.prototype.slice
305
+ .call(value, 0, 16)
306
+ .map((part) => typeof part === 'number' && Number.isSafeInteger(part) && part >= 0
307
+ ? part
308
+ : typeof part === 'string' && schemaFields.has(part)
309
+ ? part
310
+ : '[field]');
311
+ }
312
+ catch {
313
+ return [];
314
+ }
315
+ }
316
+ function safeCitationDetails(value) {
317
+ return {
318
+ path: citationPath(ownData(value, 'path')),
319
+ ref: safeEvidenceRef(ownData(value, 'ref')),
320
+ refHash: safeRefHash(ownData(value, 'refHash')),
321
+ };
322
+ }
323
+ function validationCauses(error) {
324
+ const causes = [];
325
+ const causeNames = new Set([
326
+ 'AI_InvalidToolInputError',
327
+ 'AI_NoSuchToolError',
328
+ 'AI_JSONParseError',
329
+ 'SyntaxError',
330
+ 'AI_TypeValidationError',
331
+ 'ZodError',
332
+ ]);
333
+ let current = error, issues = [];
334
+ for (let depth = 0; current && typeof current === 'object' && depth < 5; depth++) {
335
+ const value = current;
336
+ if (typeof value.name === 'string' && causeNames.has(value.name))
337
+ causes.push(value.name);
338
+ if (Array.isArray(value.issues)) {
339
+ issues = value.issues;
340
+ break;
341
+ }
342
+ current = value.cause;
343
+ }
344
+ return { causes, issues };
345
+ }
346
+ /** Emit only finite package-owned categories and known schema field names, never rejected content. */
347
+ export function submissionDiagnostics(error) {
348
+ try {
349
+ if (error instanceof CitationError) {
350
+ const rawCode = ownData(error, 'code');
351
+ return {
352
+ code: typeof rawCode === 'string' && citationCodes.has(rawCode) ? rawCode : 'phase_validation_failed',
353
+ details: { validation: [], citation: safeCitationDetails(error) },
354
+ };
355
+ }
356
+ }
357
+ catch {
358
+ return { code: 'phase_validation_failed', details: { validation: [] } };
359
+ }
360
+ const originalMessage = error instanceof Error ? error.message : '';
361
+ const message = originalMessage.replace(/^citations\.[0-9]+: /u, '');
362
+ let code = validatorCodes.has(message) ? message : 'phase_validation_failed';
363
+ const prefixes = [
364
+ ['Unknown assessmentId', 'result_identity_invalid'],
365
+ ['Duplicate/stale', 'result_revision_invalid'],
366
+ ['Result shape invalid', 'phase_shape_invalid'],
367
+ ['Unknown evidence ref', 'citation_unknown'],
368
+ ['Quote absent', 'citation_absent'],
369
+ ['Quote ambiguous', 'citation_ambiguous'],
370
+ ['Cite an exact governing', 'result_governing_passage_required'],
371
+ ['Evaluated judgments require', 'result_evidence_or_score_invalid'],
372
+ ['Findings need', 'result_correction_required'],
373
+ ['Pinned publication unavailable', 'result_publication_unavailable'],
374
+ ['A supported unmet criterion needs', 'result_finding_score_inconsistent'],
375
+ ['Adherence below 2 requires', 'result_unmet_finding_required'],
376
+ ['Unresolved/nonapplicable judgments', 'result_disposition_score_inconsistent'],
377
+ ];
378
+ for (const [prefix, category] of prefixes)
379
+ if (message.startsWith(prefix))
380
+ code = category;
381
+ const { causes, issues } = validationCauses(error);
382
+ if (causes.includes('AI_JSONParseError') || causes.includes('SyntaxError'))
383
+ code = 'phase_json_invalid';
384
+ else if (causes.includes('AI_NoSuchToolError'))
385
+ code = 'phase_tool_unknown';
386
+ else if (causes.includes('AI_InvalidToolInputError'))
387
+ code = 'phase_tool_input_invalid';
388
+ if (issues.length)
389
+ code = 'phase_shape_invalid';
390
+ const validation = issues.slice(0, 16).map((raw) => {
391
+ const issue = raw && typeof raw === 'object' ? raw : {};
392
+ return {
393
+ code: typeof issue.code === 'string' && schemaCodes.has(issue.code) ? issue.code : 'other',
394
+ path: Array.isArray(issue.path)
395
+ ? issue.path
396
+ .slice(0, 16)
397
+ .map((part) => typeof part === 'number' && Number.isSafeInteger(part)
398
+ ? part
399
+ : typeof part === 'string' && schemaFields.has(part)
400
+ ? part
401
+ : '[field]')
402
+ : [],
403
+ };
404
+ });
405
+ return { code, details: { validation, ...(causes.length ? { causes } : {}) } };
406
+ }
407
+ /** AI SDK 7 stringifies invalid tool-call errors in step.content; original calls retain structured causes. */
408
+ export function submissionToolDiagnostics(toolError, toolCalls) {
409
+ const original = toolCalls.find((call) => call.toolCallId === toolError.toolCallId);
410
+ const error = original && 'error' in original && original.error !== undefined ? original.error : toolError.error;
411
+ return submissionDiagnostics(error);
412
+ }
@@ -0,0 +1,73 @@
1
+ import { z } from 'zod';
2
+ import type { Publication } from './contracts.js';
3
+ import type { ReadResult } from './source.js';
4
+ export declare const selectionSchema: z.ZodObject<{
5
+ ref: z.ZodString;
6
+ startSegment: z.ZodNumber;
7
+ endSegment: z.ZodNumber;
8
+ }, z.core.$strict>;
9
+ export declare const citationSchema: z.ZodObject<{
10
+ ref: z.ZodString;
11
+ quote: z.ZodString;
12
+ start: z.ZodOptional<z.ZodNumber>;
13
+ end: z.ZodOptional<z.ZodNumber>;
14
+ }, z.core.$strict>;
15
+ export type CitationFailureCode = 'citation_unknown' | 'citation_absent' | 'citation_ambiguous' | 'citation_range_invalid';
16
+ /** Content-free failure identity; rejected quotations never enter exception messages or diagnostics. */
17
+ export declare class CitationError extends Error {
18
+ readonly code: CitationFailureCode;
19
+ readonly path: Array<string | number>;
20
+ readonly ref: string | null;
21
+ readonly refHash: string;
22
+ constructor(code: CitationFailureCode, ref: string, path?: Array<string | number>);
23
+ }
24
+ export type Evidence = {
25
+ ref: string;
26
+ text: string;
27
+ provenance: Record<string, unknown>;
28
+ fragments?: ReadResult['fragments'];
29
+ };
30
+ export type Citation = z.infer<typeof citationSchema> & {
31
+ start: number;
32
+ end: number;
33
+ provenance: Record<string, unknown>;
34
+ location?: {
35
+ line: number;
36
+ column: number;
37
+ };
38
+ };
39
+ /** A single source of text for model inputs: no duplicate full-text representation. */
40
+ export declare function modelEvidence(record: Evidence): {
41
+ ref: string;
42
+ provenance: Record<string, unknown>;
43
+ segments: {
44
+ index: number;
45
+ text: string;
46
+ }[];
47
+ };
48
+ /** Model-only handles and compact metadata; the ledger and audit artifacts stay canonical. */
49
+ export declare class ModelIdentifiers {
50
+ private readonly evidence;
51
+ private readonly handles;
52
+ private readonly canonical;
53
+ private evidenceCount;
54
+ constructor(assessmentIds: string[], evidence: Record<string, unknown>);
55
+ private register;
56
+ private evidenceHandle;
57
+ private translate;
58
+ toModel<T>(value: T): T;
59
+ /** Audit-only identity map; never includes the source text behind an evidence reference. */
60
+ snapshot(): Record<string, string>;
61
+ fromModel<T>(value: T): T;
62
+ }
63
+ export declare const MAX_SUBMISSION_QUOTE_BYTES: number;
64
+ /** The caller validates a selection-only model schema before this deterministic transformation. */
65
+ export declare function resolveCitationSelections(input: unknown, ledger: EvidenceLedger, path?: Array<string | number>): unknown;
66
+ export declare class EvidenceLedger {
67
+ readonly records: Record<string, Evidence>;
68
+ add(text: string, provenance: Record<string, unknown>, fragments?: ReadResult['fragments']): Evidence;
69
+ publication(publication: Publication): Evidence[];
70
+ source(result: ReadResult): Evidence | null;
71
+ select(input: z.infer<typeof selectionSchema>, path?: Array<string | number>, remainingBytes?: number): Citation;
72
+ cite(input: z.infer<typeof citationSchema>, path?: Array<string | number>): Citation;
73
+ }