jsonl-audit-kit 0.0.0-stage → 0.1.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +42 -0
- package/CONTRIBUTING.md +71 -0
- package/LICENSE +21 -0
- package/README.md +364 -2
- package/bin/jsonl-audit.js +259 -0
- package/docs/api.md +185 -0
- package/docs/design-decisions.md +128 -0
- package/docs/research.md +176 -0
- package/examples/invalid.jsonl +5 -0
- package/examples/usage.js +94 -0
- package/examples/valid.jsonl +5 -0
- package/package.json +60 -4
- package/src/diagnostics.js +124 -0
- package/src/index.js +99 -0
- package/src/utils/validate-options.js +68 -0
- package/src/validator.js +153 -0
- package/types/index.d.ts +195 -0
package/src/validator.js
ADDED
|
@@ -0,0 +1,153 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* validator.js
|
|
3
|
+
*
|
|
4
|
+
* Core JSONL auditing engine.
|
|
5
|
+
*
|
|
6
|
+
* Reads a readable stream line-by-line using node:readline.
|
|
7
|
+
* Each non-empty line is independently parsed as JSON.
|
|
8
|
+
* Validation rules are applied to each parsed record.
|
|
9
|
+
* Diagnostics are collected up to the configured maximum.
|
|
10
|
+
*
|
|
11
|
+
* Memory behaviour:
|
|
12
|
+
* - readline buffers one line at a time (bounded by the longest single line).
|
|
13
|
+
* - Diagnostics are bounded by opts.maxErrors stored entries.
|
|
14
|
+
* - Total error count is tracked separately from stored diagnostics.
|
|
15
|
+
*/
|
|
16
|
+
|
|
17
|
+
import { createInterface } from 'node:readline';
|
|
18
|
+
import {
|
|
19
|
+
invalidJson,
|
|
20
|
+
blankLine,
|
|
21
|
+
notAnObject,
|
|
22
|
+
notAnArray,
|
|
23
|
+
missingKey,
|
|
24
|
+
lineTooLong,
|
|
25
|
+
} from './diagnostics.js';
|
|
26
|
+
|
|
27
|
+
/**
|
|
28
|
+
* Determine the JavaScript type name suitable for a diagnostic message.
|
|
29
|
+
*
|
|
30
|
+
* @param {unknown} value
|
|
31
|
+
* @returns {string}
|
|
32
|
+
*/
|
|
33
|
+
function typeName(value) {
|
|
34
|
+
if (value === null) return 'null';
|
|
35
|
+
if (Array.isArray(value)) return 'array';
|
|
36
|
+
return typeof value;
|
|
37
|
+
}
|
|
38
|
+
|
|
39
|
+
/**
|
|
40
|
+
* Process a single non-blank line.
|
|
41
|
+
*
|
|
42
|
+
* Returns an array of zero or more diagnostics for that line.
|
|
43
|
+
*
|
|
44
|
+
* @param {string} rawLine - The raw text of the line (no trailing newline).
|
|
45
|
+
* @param {number} lineNumber - 1-based line number.
|
|
46
|
+
* @param {object} opts - Fully-resolved audit options.
|
|
47
|
+
* @returns {Array<object>}
|
|
48
|
+
*/
|
|
49
|
+
export function validateLine(rawLine, lineNumber, opts) {
|
|
50
|
+
const diagnostics = [];
|
|
51
|
+
|
|
52
|
+
// ── Size check ───────────────────────────────────────────────────────────
|
|
53
|
+
if (opts.maxLineBytes !== null) {
|
|
54
|
+
const byteLen = Buffer.byteLength(rawLine, 'utf8');
|
|
55
|
+
if (byteLen > opts.maxLineBytes) {
|
|
56
|
+
diagnostics.push(lineTooLong(lineNumber, byteLen, opts.maxLineBytes));
|
|
57
|
+
// We still attempt to parse the line; both errors may be reported.
|
|
58
|
+
}
|
|
59
|
+
}
|
|
60
|
+
|
|
61
|
+
// ── JSON parse ────────────────────────────────────────────────────────────
|
|
62
|
+
let parsed;
|
|
63
|
+
try {
|
|
64
|
+
parsed = JSON.parse(rawLine);
|
|
65
|
+
} catch (err) {
|
|
66
|
+
diagnostics.push(invalidJson(lineNumber, err.message));
|
|
67
|
+
return diagnostics; // Cannot apply structural rules to un-parseable JSON.
|
|
68
|
+
}
|
|
69
|
+
|
|
70
|
+
// ── Structural rules ──────────────────────────────────────────────────────
|
|
71
|
+
if (opts.requireObject) {
|
|
72
|
+
if (parsed === null || typeof parsed !== 'object' || Array.isArray(parsed)) {
|
|
73
|
+
diagnostics.push(notAnObject(lineNumber, typeName(parsed)));
|
|
74
|
+
return diagnostics; // Required-key checks are only meaningful for objects.
|
|
75
|
+
}
|
|
76
|
+
}
|
|
77
|
+
|
|
78
|
+
if (opts.requireArray) {
|
|
79
|
+
if (!Array.isArray(parsed)) {
|
|
80
|
+
diagnostics.push(notAnArray(lineNumber, typeName(parsed)));
|
|
81
|
+
return diagnostics;
|
|
82
|
+
}
|
|
83
|
+
}
|
|
84
|
+
|
|
85
|
+
// ── Required-key checks ───────────────────────────────────────────────────
|
|
86
|
+
if (opts.requiredKeys.length > 0) {
|
|
87
|
+
if (parsed !== null && typeof parsed === 'object' && !Array.isArray(parsed)) {
|
|
88
|
+
for (const key of opts.requiredKeys) {
|
|
89
|
+
if (!Object.prototype.hasOwnProperty.call(parsed, key)) {
|
|
90
|
+
diagnostics.push(missingKey(lineNumber, key));
|
|
91
|
+
}
|
|
92
|
+
}
|
|
93
|
+
}
|
|
94
|
+
}
|
|
95
|
+
|
|
96
|
+
return diagnostics;
|
|
97
|
+
}
|
|
98
|
+
|
|
99
|
+
/**
|
|
100
|
+
* Audit a readable stream containing JSONL data.
|
|
101
|
+
*
|
|
102
|
+
* @param {import('node:stream').Readable} stream - Input stream.
|
|
103
|
+
* @param {object} opts - Fully-resolved audit options (from validateOptions).
|
|
104
|
+
* @returns {Promise<import('../types/index.d.ts').AuditResult>}
|
|
105
|
+
*/
|
|
106
|
+
export async function auditStream(stream, opts) {
|
|
107
|
+
const rl = createInterface({
|
|
108
|
+
input: stream,
|
|
109
|
+
crlfDelay: Infinity, // Handle both \n and \r\n line endings.
|
|
110
|
+
});
|
|
111
|
+
|
|
112
|
+
let totalLines = 0;
|
|
113
|
+
let blankLines = 0;
|
|
114
|
+
let errorCount = 0;
|
|
115
|
+
const errors = [];
|
|
116
|
+
|
|
117
|
+
for await (const rawLine of rl) {
|
|
118
|
+
totalLines += 1;
|
|
119
|
+
|
|
120
|
+
// ── Blank-line handling ───────────────────────────────────────────────
|
|
121
|
+
if (rawLine.trim() === '') {
|
|
122
|
+
blankLines += 1;
|
|
123
|
+
if (opts.reportBlankLines) {
|
|
124
|
+
errorCount += 1;
|
|
125
|
+
if (errors.length < opts.maxErrors) {
|
|
126
|
+
errors.push(blankLine(totalLines));
|
|
127
|
+
}
|
|
128
|
+
}
|
|
129
|
+
continue;
|
|
130
|
+
}
|
|
131
|
+
|
|
132
|
+
// ── Validate the line ─────────────────────────────────────────────────
|
|
133
|
+
const lineDiagnostics = validateLine(rawLine, totalLines, opts);
|
|
134
|
+
|
|
135
|
+
for (const diagnostic of lineDiagnostics) {
|
|
136
|
+
errorCount += 1;
|
|
137
|
+
if (errors.length < opts.maxErrors) {
|
|
138
|
+
errors.push(diagnostic);
|
|
139
|
+
}
|
|
140
|
+
}
|
|
141
|
+
}
|
|
142
|
+
|
|
143
|
+
const truncated = errors.length < errorCount;
|
|
144
|
+
|
|
145
|
+
return {
|
|
146
|
+
valid: errorCount === 0,
|
|
147
|
+
totalLines,
|
|
148
|
+
blankLines,
|
|
149
|
+
errorCount,
|
|
150
|
+
errors,
|
|
151
|
+
truncated,
|
|
152
|
+
};
|
|
153
|
+
}
|
package/types/index.d.ts
ADDED
|
@@ -0,0 +1,195 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* TypeScript declarations for jsonl-audit-kit
|
|
3
|
+
*
|
|
4
|
+
* These declarations describe the public JavaScript API.
|
|
5
|
+
* The implementation is in plain JavaScript (ESM).
|
|
6
|
+
*/
|
|
7
|
+
|
|
8
|
+
import type { Readable } from 'node:stream';
|
|
9
|
+
|
|
10
|
+
// ── Enumerations ──────────────────────────────────────────────────────────────
|
|
11
|
+
|
|
12
|
+
/**
|
|
13
|
+
* String literals identifying the kind of problem found in a JSONL record.
|
|
14
|
+
*/
|
|
15
|
+
export declare const DiagnosticType: {
|
|
16
|
+
readonly INVALID_JSON: 'INVALID_JSON';
|
|
17
|
+
readonly BLANK_LINE: 'BLANK_LINE';
|
|
18
|
+
readonly NOT_AN_OBJECT: 'NOT_AN_OBJECT';
|
|
19
|
+
readonly NOT_AN_ARRAY: 'NOT_AN_ARRAY';
|
|
20
|
+
readonly MISSING_KEY: 'MISSING_KEY';
|
|
21
|
+
readonly LINE_TOO_LONG: 'LINE_TOO_LONG';
|
|
22
|
+
};
|
|
23
|
+
|
|
24
|
+
export type DiagnosticTypeName = typeof DiagnosticType[keyof typeof DiagnosticType];
|
|
25
|
+
|
|
26
|
+
// ── Diagnostic ────────────────────────────────────────────────────────────────
|
|
27
|
+
|
|
28
|
+
/**
|
|
29
|
+
* A single diagnostic describing one problem in one line of a JSONL file.
|
|
30
|
+
*/
|
|
31
|
+
export interface Diagnostic {
|
|
32
|
+
/** 1-based line number where the problem occurred. */
|
|
33
|
+
line: number;
|
|
34
|
+
|
|
35
|
+
/** Category of the problem. One of the DiagnosticType values. */
|
|
36
|
+
type: DiagnosticTypeName;
|
|
37
|
+
|
|
38
|
+
/** Human-readable description of the problem. */
|
|
39
|
+
message: string;
|
|
40
|
+
|
|
41
|
+
/**
|
|
42
|
+
* Present on MISSING_KEY diagnostics: the name of the key that was absent.
|
|
43
|
+
*/
|
|
44
|
+
key?: string;
|
|
45
|
+
|
|
46
|
+
/**
|
|
47
|
+
* Present on LINE_TOO_LONG diagnostics: actual byte length of the line.
|
|
48
|
+
*/
|
|
49
|
+
length?: number;
|
|
50
|
+
|
|
51
|
+
/**
|
|
52
|
+
* Present on LINE_TOO_LONG diagnostics: the configured byte limit.
|
|
53
|
+
*/
|
|
54
|
+
max?: number;
|
|
55
|
+
}
|
|
56
|
+
|
|
57
|
+
// ── Options ───────────────────────────────────────────────────────────────────
|
|
58
|
+
|
|
59
|
+
/**
|
|
60
|
+
* Configuration options accepted by auditFile and auditStream.
|
|
61
|
+
*/
|
|
62
|
+
export interface AuditOptions {
|
|
63
|
+
/**
|
|
64
|
+
* When true, every record must be a JSON object ({...}).
|
|
65
|
+
* Records that are arrays, strings, numbers, booleans, or null are flagged
|
|
66
|
+
* with a NOT_AN_OBJECT diagnostic.
|
|
67
|
+
*
|
|
68
|
+
* Cannot be combined with requireArray.
|
|
69
|
+
*
|
|
70
|
+
* @default false
|
|
71
|
+
*/
|
|
72
|
+
requireObject?: boolean;
|
|
73
|
+
|
|
74
|
+
/**
|
|
75
|
+
* When true, every record must be a JSON array ([...]).
|
|
76
|
+
* Records that are objects, strings, numbers, booleans, or null are flagged
|
|
77
|
+
* with a NOT_AN_ARRAY diagnostic.
|
|
78
|
+
*
|
|
79
|
+
* Cannot be combined with requireObject.
|
|
80
|
+
*
|
|
81
|
+
* @default false
|
|
82
|
+
*/
|
|
83
|
+
requireArray?: boolean;
|
|
84
|
+
|
|
85
|
+
/**
|
|
86
|
+
* A list of key names that must be present at the top level of every
|
|
87
|
+
* JSON object record. Only applied when the record is a JSON object.
|
|
88
|
+
*
|
|
89
|
+
* @default []
|
|
90
|
+
*/
|
|
91
|
+
requiredKeys?: string[];
|
|
92
|
+
|
|
93
|
+
/**
|
|
94
|
+
* When true, blank (whitespace-only) lines are reported as errors.
|
|
95
|
+
* When false (the default), blank lines are silently skipped.
|
|
96
|
+
*
|
|
97
|
+
* @default false
|
|
98
|
+
*/
|
|
99
|
+
reportBlankLines?: boolean;
|
|
100
|
+
|
|
101
|
+
/**
|
|
102
|
+
* Maximum number of diagnostics to retain in the result.errors array.
|
|
103
|
+
*
|
|
104
|
+
* Scanning always continues to completion regardless of this limit.
|
|
105
|
+
* The result.errorCount field reflects the total number of problems found.
|
|
106
|
+
* result.truncated will be true when errorCount exceeds maxErrors.
|
|
107
|
+
*
|
|
108
|
+
* @default 100
|
|
109
|
+
*/
|
|
110
|
+
maxErrors?: number;
|
|
111
|
+
|
|
112
|
+
/**
|
|
113
|
+
* When set, any line whose UTF-8 byte length exceeds this value is flagged
|
|
114
|
+
* with a LINE_TOO_LONG diagnostic. The line is still parsed after the check.
|
|
115
|
+
*
|
|
116
|
+
* @default null (no limit)
|
|
117
|
+
*/
|
|
118
|
+
maxLineBytes?: number | null;
|
|
119
|
+
}
|
|
120
|
+
|
|
121
|
+
// ── Result ────────────────────────────────────────────────────────────────────
|
|
122
|
+
|
|
123
|
+
/**
|
|
124
|
+
* The result returned by auditFile and auditStream.
|
|
125
|
+
*/
|
|
126
|
+
export interface AuditResult {
|
|
127
|
+
/**
|
|
128
|
+
* true if no errors were found; false if any validation problem was detected.
|
|
129
|
+
*/
|
|
130
|
+
valid: boolean;
|
|
131
|
+
|
|
132
|
+
/**
|
|
133
|
+
* Total number of lines (including blank lines) in the input.
|
|
134
|
+
*/
|
|
135
|
+
totalLines: number;
|
|
136
|
+
|
|
137
|
+
/**
|
|
138
|
+
* Number of blank (whitespace-only) lines encountered.
|
|
139
|
+
*/
|
|
140
|
+
blankLines: number;
|
|
141
|
+
|
|
142
|
+
/**
|
|
143
|
+
* Total number of validation problems found.
|
|
144
|
+
* May exceed errors.length when truncated is true.
|
|
145
|
+
*/
|
|
146
|
+
errorCount: number;
|
|
147
|
+
|
|
148
|
+
/**
|
|
149
|
+
* Stored diagnostics. Contains at most maxErrors entries.
|
|
150
|
+
* When truncated is true, additional errors exist beyond this list.
|
|
151
|
+
*/
|
|
152
|
+
errors: Diagnostic[];
|
|
153
|
+
|
|
154
|
+
/**
|
|
155
|
+
* true when errorCount exceeds maxErrors and not all errors are stored.
|
|
156
|
+
*/
|
|
157
|
+
truncated: boolean;
|
|
158
|
+
}
|
|
159
|
+
|
|
160
|
+
// ── Public API ────────────────────────────────────────────────────────────────
|
|
161
|
+
|
|
162
|
+
/**
|
|
163
|
+
* Audit a JSONL file at the given path.
|
|
164
|
+
*
|
|
165
|
+
* The file is read using Node.js streams and readline — it is never fully
|
|
166
|
+
* loaded into memory, making this suitable for files of any size.
|
|
167
|
+
*
|
|
168
|
+
* @param filePath - Absolute or relative path to the JSONL file.
|
|
169
|
+
* @param options - Optional audit configuration.
|
|
170
|
+
* @returns A promise that resolves to an AuditResult.
|
|
171
|
+
*
|
|
172
|
+
* @throws {TypeError} When filePath is not a string, or options are invalid.
|
|
173
|
+
* @throws {Error} When the file cannot be opened (ENOENT, EACCES, EISDIR).
|
|
174
|
+
*/
|
|
175
|
+
export declare function auditFile(
|
|
176
|
+
filePath: string,
|
|
177
|
+
options?: AuditOptions,
|
|
178
|
+
): Promise<AuditResult>;
|
|
179
|
+
|
|
180
|
+
/**
|
|
181
|
+
* Audit a JSONL readable stream.
|
|
182
|
+
*
|
|
183
|
+
* Use this when you already have a stream object — for example,
|
|
184
|
+
* process.stdin, an HTTP response body, or a custom readable.
|
|
185
|
+
*
|
|
186
|
+
* @param stream - A readable Node.js stream containing UTF-8 JSONL data.
|
|
187
|
+
* @param options - Optional audit configuration.
|
|
188
|
+
* @returns A promise that resolves to an AuditResult.
|
|
189
|
+
*
|
|
190
|
+
* @throws {TypeError} When stream is not a readable, or options are invalid.
|
|
191
|
+
*/
|
|
192
|
+
export declare function auditStream(
|
|
193
|
+
stream: Readable,
|
|
194
|
+
options?: AuditOptions,
|
|
195
|
+
): Promise<AuditResult>;
|