jsonl-audit-kit 0.0.0-stage → 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +42 -0
- package/CONTRIBUTING.md +71 -0
- package/LICENSE +21 -0
- package/README.md +364 -2
- package/bin/jsonl-audit.js +259 -0
- package/docs/api.md +185 -0
- package/docs/design-decisions.md +128 -0
- package/docs/research.md +176 -0
- package/examples/invalid.jsonl +5 -0
- package/examples/usage.js +94 -0
- package/examples/valid.jsonl +5 -0
- package/package.json +60 -4
- package/src/diagnostics.js +124 -0
- package/src/index.js +99 -0
- package/src/utils/validate-options.js +68 -0
- package/src/validator.js +153 -0
- package/types/index.d.ts +195 -0
|
@@ -0,0 +1,259 @@
|
|
|
1
|
+
#!/usr/bin/env node
|
|
2
|
+
/**
|
|
3
|
+
* jsonl-audit.js — Command-line interface for jsonl-audit-kit
|
|
4
|
+
*
|
|
5
|
+
* Usage:
|
|
6
|
+
* jsonl-audit <file> Audit a JSONL file
|
|
7
|
+
* jsonl-audit - Read from stdin
|
|
8
|
+
* cat data.jsonl | jsonl-audit - Pipe data in
|
|
9
|
+
*
|
|
10
|
+
* Options:
|
|
11
|
+
* --require-object Require each record to be a JSON object
|
|
12
|
+
* --require-array Require each record to be a JSON array
|
|
13
|
+
* --required <keys> Comma-separated list of required top-level keys
|
|
14
|
+
* --report-blank-lines Treat blank lines as errors
|
|
15
|
+
* --max-errors <n> Maximum number of errors to store (default: 100)
|
|
16
|
+
* --max-line-bytes <n> Reject lines longer than this many bytes
|
|
17
|
+
* --json Output results as a JSON object
|
|
18
|
+
* --version Print version and exit
|
|
19
|
+
* --help Show usage and exit
|
|
20
|
+
*
|
|
21
|
+
* Exit codes:
|
|
22
|
+
* 0 File is valid (all records passed)
|
|
23
|
+
* 1 File is invalid (one or more records failed)
|
|
24
|
+
* 2 Operational error (file not found, bad arguments, etc.)
|
|
25
|
+
*/
|
|
26
|
+
|
|
27
|
+
import { readFileSync } from 'node:fs';
|
|
28
|
+
import { createRequire } from 'node:module';
|
|
29
|
+
import { resolve, dirname } from 'node:path';
|
|
30
|
+
import { fileURLToPath } from 'node:url';
|
|
31
|
+
import { auditFile, auditStream } from '../src/index.js';
|
|
32
|
+
|
|
33
|
+
const __dirname = dirname(fileURLToPath(import.meta.url));
|
|
34
|
+
|
|
35
|
+
// ── Load package.json for --version ─────────────────────────────────────────
|
|
36
|
+
let pkg;
|
|
37
|
+
try {
|
|
38
|
+
const pkgPath = resolve(__dirname, '../package.json');
|
|
39
|
+
pkg = JSON.parse(readFileSync(pkgPath, 'utf8'));
|
|
40
|
+
} catch {
|
|
41
|
+
pkg = { version: '0.0.0' };
|
|
42
|
+
}
|
|
43
|
+
|
|
44
|
+
// ── Argument parsing ─────────────────────────────────────────────────────────
|
|
45
|
+
|
|
46
|
+
const EXIT_VALID = 0;
|
|
47
|
+
const EXIT_INVALID = 1;
|
|
48
|
+
const EXIT_ERROR = 2;
|
|
49
|
+
|
|
50
|
+
function showHelp() {
|
|
51
|
+
process.stdout.write(`
|
|
52
|
+
jsonl-audit — Audit JSONL (JSON Lines) files for invalid records
|
|
53
|
+
|
|
54
|
+
USAGE
|
|
55
|
+
jsonl-audit <file> Audit a file
|
|
56
|
+
jsonl-audit - Read from stdin
|
|
57
|
+
cat data.jsonl | jsonl-audit - Pipe data in
|
|
58
|
+
|
|
59
|
+
OPTIONS
|
|
60
|
+
--require-object Require each record to be a JSON object ({...})
|
|
61
|
+
--require-array Require each record to be a JSON array ([...])
|
|
62
|
+
--required <k1,k2,...> Comma-separated required top-level key names
|
|
63
|
+
--report-blank-lines Treat blank lines as validation errors
|
|
64
|
+
--max-errors <n> Max stored errors (default: 100). Scanning always
|
|
65
|
+
continues to completion; excess errors are counted
|
|
66
|
+
but not stored.
|
|
67
|
+
--max-line-bytes <n> Flag lines longer than n bytes
|
|
68
|
+
--json Output results as JSON (machine-readable)
|
|
69
|
+
--version Print version and exit
|
|
70
|
+
--help Show this help and exit
|
|
71
|
+
|
|
72
|
+
EXIT CODES
|
|
73
|
+
0 All records are valid
|
|
74
|
+
1 One or more records are invalid
|
|
75
|
+
2 Operational error (file not found, permission denied, bad arguments)
|
|
76
|
+
|
|
77
|
+
EXAMPLES
|
|
78
|
+
jsonl-audit events.jsonl
|
|
79
|
+
jsonl-audit events.jsonl --require-object --required id,event
|
|
80
|
+
jsonl-audit events.jsonl --max-errors 10 --json
|
|
81
|
+
cat events.jsonl | jsonl-audit -
|
|
82
|
+
`.trimStart());
|
|
83
|
+
}
|
|
84
|
+
|
|
85
|
+
/**
|
|
86
|
+
* Parse argv into a structured object.
|
|
87
|
+
* Returns { target, options, jsonOutput, error }
|
|
88
|
+
*/
|
|
89
|
+
function parseArgs(argv) {
|
|
90
|
+
const args = argv.slice(2);
|
|
91
|
+
let target = null;
|
|
92
|
+
let jsonOutput = false;
|
|
93
|
+
const opts = {};
|
|
94
|
+
let error = null;
|
|
95
|
+
|
|
96
|
+
let i = 0;
|
|
97
|
+
while (i < args.length) {
|
|
98
|
+
const arg = args[i];
|
|
99
|
+
|
|
100
|
+
if (arg === '--help' || arg === '-h') {
|
|
101
|
+
showHelp();
|
|
102
|
+
process.exit(EXIT_VALID);
|
|
103
|
+
}
|
|
104
|
+
|
|
105
|
+
if (arg === '--version' || arg === '-v') {
|
|
106
|
+
process.stdout.write(`jsonl-audit-kit v${pkg.version}\n`);
|
|
107
|
+
process.exit(EXIT_VALID);
|
|
108
|
+
}
|
|
109
|
+
|
|
110
|
+
if (arg === '--json') {
|
|
111
|
+
jsonOutput = true;
|
|
112
|
+
i++;
|
|
113
|
+
continue;
|
|
114
|
+
}
|
|
115
|
+
|
|
116
|
+
if (arg === '--require-object') {
|
|
117
|
+
opts.requireObject = true;
|
|
118
|
+
i++;
|
|
119
|
+
continue;
|
|
120
|
+
}
|
|
121
|
+
|
|
122
|
+
if (arg === '--require-array') {
|
|
123
|
+
opts.requireArray = true;
|
|
124
|
+
i++;
|
|
125
|
+
continue;
|
|
126
|
+
}
|
|
127
|
+
|
|
128
|
+
if (arg === '--report-blank-lines') {
|
|
129
|
+
opts.reportBlankLines = true;
|
|
130
|
+
i++;
|
|
131
|
+
continue;
|
|
132
|
+
}
|
|
133
|
+
|
|
134
|
+
if (arg === '--required') {
|
|
135
|
+
const val = args[i + 1];
|
|
136
|
+
if (!val || val.startsWith('--')) {
|
|
137
|
+
error = '--required requires a comma-separated list of key names';
|
|
138
|
+
break;
|
|
139
|
+
}
|
|
140
|
+
opts.requiredKeys = val.split(',').map((k) => k.trim()).filter(Boolean);
|
|
141
|
+
i += 2;
|
|
142
|
+
continue;
|
|
143
|
+
}
|
|
144
|
+
|
|
145
|
+
if (arg === '--max-errors') {
|
|
146
|
+
const val = parseInt(args[i + 1], 10);
|
|
147
|
+
if (!Number.isInteger(val) || val < 1) {
|
|
148
|
+
error = '--max-errors requires a positive integer';
|
|
149
|
+
break;
|
|
150
|
+
}
|
|
151
|
+
opts.maxErrors = val;
|
|
152
|
+
i += 2;
|
|
153
|
+
continue;
|
|
154
|
+
}
|
|
155
|
+
|
|
156
|
+
if (arg === '--max-line-bytes') {
|
|
157
|
+
const val = parseInt(args[i + 1], 10);
|
|
158
|
+
if (!Number.isInteger(val) || val < 1) {
|
|
159
|
+
error = '--max-line-bytes requires a positive integer';
|
|
160
|
+
break;
|
|
161
|
+
}
|
|
162
|
+
opts.maxLineBytes = val;
|
|
163
|
+
i += 2;
|
|
164
|
+
continue;
|
|
165
|
+
}
|
|
166
|
+
|
|
167
|
+
if (arg.startsWith('--')) {
|
|
168
|
+
error = `Unknown option: ${arg}`;
|
|
169
|
+
break;
|
|
170
|
+
}
|
|
171
|
+
|
|
172
|
+
// Positional: target file or '-' for stdin
|
|
173
|
+
if (target !== null) {
|
|
174
|
+
error = 'Too many positional arguments — expected one file path or "-"';
|
|
175
|
+
break;
|
|
176
|
+
}
|
|
177
|
+
target = arg;
|
|
178
|
+
i++;
|
|
179
|
+
}
|
|
180
|
+
|
|
181
|
+
return { target, options: opts, jsonOutput, error };
|
|
182
|
+
}
|
|
183
|
+
|
|
184
|
+
// ── Output helpers ────────────────────────────────────────────────────────────
|
|
185
|
+
|
|
186
|
+
function humanOutput(result, source) {
|
|
187
|
+
const label = source === '-' ? '<stdin>' : source;
|
|
188
|
+
|
|
189
|
+
if (result.valid) {
|
|
190
|
+
process.stdout.write(
|
|
191
|
+
`✓ ${label}: valid — ${result.totalLines} line(s) checked, 0 errors\n`,
|
|
192
|
+
);
|
|
193
|
+
return;
|
|
194
|
+
}
|
|
195
|
+
|
|
196
|
+
process.stderr.write(
|
|
197
|
+
`✗ ${label}: INVALID — ${result.errorCount} error(s) in ${result.totalLines} line(s)\n`,
|
|
198
|
+
);
|
|
199
|
+
|
|
200
|
+
if (result.truncated) {
|
|
201
|
+
process.stderr.write(
|
|
202
|
+
` (showing first ${result.errors.length} of ${result.errorCount} errors)\n`,
|
|
203
|
+
);
|
|
204
|
+
}
|
|
205
|
+
|
|
206
|
+
for (const diag of result.errors) {
|
|
207
|
+
const keyInfo = diag.key ? ` [key: ${diag.key}]` : '';
|
|
208
|
+
process.stderr.write(
|
|
209
|
+
` Line ${diag.line}: [${diag.type}] ${diag.message}${keyInfo}\n`,
|
|
210
|
+
);
|
|
211
|
+
}
|
|
212
|
+
}
|
|
213
|
+
|
|
214
|
+
function jsonOutputFn(result, source) {
|
|
215
|
+
const output = {
|
|
216
|
+
source: source === '-' ? '<stdin>' : source,
|
|
217
|
+
...result,
|
|
218
|
+
};
|
|
219
|
+
process.stdout.write(JSON.stringify(output, null, 2) + '\n');
|
|
220
|
+
}
|
|
221
|
+
|
|
222
|
+
// ── Main ──────────────────────────────────────────────────────────────────────
|
|
223
|
+
|
|
224
|
+
async function main() {
|
|
225
|
+
const { target, options, jsonOutput, error } = parseArgs(process.argv);
|
|
226
|
+
|
|
227
|
+
if (error) {
|
|
228
|
+
process.stderr.write(`Error: ${error}\n\nRun with --help for usage.\n`);
|
|
229
|
+
process.exit(EXIT_ERROR);
|
|
230
|
+
}
|
|
231
|
+
|
|
232
|
+
if (target === null) {
|
|
233
|
+
showHelp();
|
|
234
|
+
process.exit(EXIT_ERROR);
|
|
235
|
+
}
|
|
236
|
+
|
|
237
|
+
let result;
|
|
238
|
+
try {
|
|
239
|
+
if (target === '-') {
|
|
240
|
+
// Read from stdin
|
|
241
|
+
result = await auditStream(process.stdin, options);
|
|
242
|
+
} else {
|
|
243
|
+
result = await auditFile(target, options);
|
|
244
|
+
}
|
|
245
|
+
} catch (err) {
|
|
246
|
+
process.stderr.write(`Error: ${err.message}\n`);
|
|
247
|
+
process.exit(EXIT_ERROR);
|
|
248
|
+
}
|
|
249
|
+
|
|
250
|
+
if (jsonOutput) {
|
|
251
|
+
jsonOutputFn(result, target);
|
|
252
|
+
} else {
|
|
253
|
+
humanOutput(result, target);
|
|
254
|
+
}
|
|
255
|
+
|
|
256
|
+
process.exit(result.valid ? EXIT_VALID : EXIT_INVALID);
|
|
257
|
+
}
|
|
258
|
+
|
|
259
|
+
main();
|
package/docs/api.md
ADDED
|
@@ -0,0 +1,185 @@
|
|
|
1
|
+
# API Reference — jsonl-audit-kit
|
|
2
|
+
|
|
3
|
+
## Functions
|
|
4
|
+
|
|
5
|
+
### `auditFile(filePath, options?)`
|
|
6
|
+
|
|
7
|
+
Audit a JSONL file at the given path.
|
|
8
|
+
|
|
9
|
+
```js
|
|
10
|
+
import { auditFile } from 'jsonl-audit-kit';
|
|
11
|
+
|
|
12
|
+
const result = await auditFile('./events.jsonl', {
|
|
13
|
+
requireObject: true,
|
|
14
|
+
requiredKeys: ['id', 'event'],
|
|
15
|
+
maxErrors: 50,
|
|
16
|
+
});
|
|
17
|
+
```
|
|
18
|
+
|
|
19
|
+
**Parameters**
|
|
20
|
+
|
|
21
|
+
| Parameter | Type | Required | Description |
|
|
22
|
+
|-----------|------|----------|-------------|
|
|
23
|
+
| `filePath` | `string` | Yes | Absolute or relative path to the JSONL file |
|
|
24
|
+
| `options` | `AuditOptions` | No | Validation options (see below) |
|
|
25
|
+
|
|
26
|
+
**Returns:** `Promise<AuditResult>`
|
|
27
|
+
|
|
28
|
+
**Throws:**
|
|
29
|
+
- `TypeError` — `filePath` is not a string, or options are invalid
|
|
30
|
+
- `Error` (code `ENOENT`) — file does not exist
|
|
31
|
+
- `Error` (code `EISDIR`) — path is a directory
|
|
32
|
+
- `Error` (code `EACCES`) — file is not readable
|
|
33
|
+
|
|
34
|
+
---
|
|
35
|
+
|
|
36
|
+
### `auditStream(stream, options?)`
|
|
37
|
+
|
|
38
|
+
Audit a readable Node.js stream containing JSONL data.
|
|
39
|
+
|
|
40
|
+
```js
|
|
41
|
+
import { auditStream } from 'jsonl-audit-kit';
|
|
42
|
+
import { createReadStream } from 'node:fs';
|
|
43
|
+
|
|
44
|
+
const stream = createReadStream('./events.jsonl', { encoding: 'utf8' });
|
|
45
|
+
const result = await auditStream(stream, { requireObject: true });
|
|
46
|
+
```
|
|
47
|
+
|
|
48
|
+
**Parameters**
|
|
49
|
+
|
|
50
|
+
| Parameter | Type | Required | Description |
|
|
51
|
+
|-----------|------|----------|-------------|
|
|
52
|
+
| `stream` | `Readable` | Yes | A readable Node.js stream |
|
|
53
|
+
| `options` | `AuditOptions` | No | Validation options (see below) |
|
|
54
|
+
|
|
55
|
+
**Returns:** `Promise<AuditResult>`
|
|
56
|
+
|
|
57
|
+
---
|
|
58
|
+
|
|
59
|
+
## Types
|
|
60
|
+
|
|
61
|
+
### `AuditOptions`
|
|
62
|
+
|
|
63
|
+
| Property | Type | Default | Description |
|
|
64
|
+
|----------|------|---------|-------------|
|
|
65
|
+
| `requireObject` | `boolean` | `false` | Require every record to be a JSON object `{...}` |
|
|
66
|
+
| `requireArray` | `boolean` | `false` | Require every record to be a JSON array `[...]` |
|
|
67
|
+
| `requiredKeys` | `string[]` | `[]` | Keys that must be present in every object record |
|
|
68
|
+
| `reportBlankLines` | `boolean` | `false` | Treat blank lines as errors |
|
|
69
|
+
| `maxErrors` | `number` | `100` | Maximum stored diagnostics (scanning always completes) |
|
|
70
|
+
| `maxLineBytes` | `number \| null` | `null` | Flag lines longer than this many bytes |
|
|
71
|
+
|
|
72
|
+
`requireObject` and `requireArray` cannot both be `true`.
|
|
73
|
+
|
|
74
|
+
---
|
|
75
|
+
|
|
76
|
+
### `AuditResult`
|
|
77
|
+
|
|
78
|
+
| Property | Type | Description |
|
|
79
|
+
|----------|------|-------------|
|
|
80
|
+
| `valid` | `boolean` | `true` if no errors were found |
|
|
81
|
+
| `totalLines` | `number` | Total number of lines read (including blank lines) |
|
|
82
|
+
| `blankLines` | `number` | Number of blank / whitespace-only lines |
|
|
83
|
+
| `errorCount` | `number` | Total number of validation errors (may exceed `errors.length`) |
|
|
84
|
+
| `errors` | `Diagnostic[]` | Stored diagnostics (at most `maxErrors` entries) |
|
|
85
|
+
| `truncated` | `boolean` | `true` when `errorCount > errors.length` |
|
|
86
|
+
|
|
87
|
+
---
|
|
88
|
+
|
|
89
|
+
### `Diagnostic`
|
|
90
|
+
|
|
91
|
+
| Property | Type | Present when |
|
|
92
|
+
|----------|------|--------------|
|
|
93
|
+
| `line` | `number` | Always — 1-based line number |
|
|
94
|
+
| `type` | `DiagnosticTypeName` | Always — category of the problem |
|
|
95
|
+
| `message` | `string` | Always — human-readable description |
|
|
96
|
+
| `key` | `string` | `MISSING_KEY` diagnostics only |
|
|
97
|
+
| `length` | `number` | `LINE_TOO_LONG` diagnostics only |
|
|
98
|
+
| `max` | `number` | `LINE_TOO_LONG` diagnostics only |
|
|
99
|
+
|
|
100
|
+
---
|
|
101
|
+
|
|
102
|
+
### `DiagnosticType`
|
|
103
|
+
|
|
104
|
+
```js
|
|
105
|
+
import { DiagnosticType } from 'jsonl-audit-kit';
|
|
106
|
+
|
|
107
|
+
DiagnosticType.INVALID_JSON // 'INVALID_JSON'
|
|
108
|
+
DiagnosticType.BLANK_LINE // 'BLANK_LINE'
|
|
109
|
+
DiagnosticType.NOT_AN_OBJECT // 'NOT_AN_OBJECT'
|
|
110
|
+
DiagnosticType.NOT_AN_ARRAY // 'NOT_AN_ARRAY'
|
|
111
|
+
DiagnosticType.MISSING_KEY // 'MISSING_KEY'
|
|
112
|
+
DiagnosticType.LINE_TOO_LONG // 'LINE_TOO_LONG'
|
|
113
|
+
```
|
|
114
|
+
|
|
115
|
+
| Constant | Meaning |
|
|
116
|
+
|----------|---------|
|
|
117
|
+
| `INVALID_JSON` | Line is not valid JSON |
|
|
118
|
+
| `BLANK_LINE` | Line is blank (only when `reportBlankLines: true`) |
|
|
119
|
+
| `NOT_AN_OBJECT` | Parsed value is not a plain object (when `requireObject: true`) |
|
|
120
|
+
| `NOT_AN_ARRAY` | Parsed value is not an array (when `requireArray: true`) |
|
|
121
|
+
| `MISSING_KEY` | A required key is absent from the top-level object |
|
|
122
|
+
| `LINE_TOO_LONG` | Line exceeds `maxLineBytes` |
|
|
123
|
+
|
|
124
|
+
---
|
|
125
|
+
|
|
126
|
+
## CLI Reference
|
|
127
|
+
|
|
128
|
+
```
|
|
129
|
+
jsonl-audit <file|-> [options]
|
|
130
|
+
```
|
|
131
|
+
|
|
132
|
+
### Options
|
|
133
|
+
|
|
134
|
+
| Flag | Argument | Description |
|
|
135
|
+
|------|----------|-------------|
|
|
136
|
+
| `--require-object` | — | Require every record to be a JSON object |
|
|
137
|
+
| `--require-array` | — | Require every record to be a JSON array |
|
|
138
|
+
| `--required` | `k1,k2,...` | Comma-separated required key names |
|
|
139
|
+
| `--report-blank-lines` | — | Treat blank lines as errors |
|
|
140
|
+
| `--max-errors` | `n` | Max stored errors (default: 100) |
|
|
141
|
+
| `--max-line-bytes` | `n` | Flag lines longer than n bytes |
|
|
142
|
+
| `--json` | — | Machine-readable JSON output to stdout |
|
|
143
|
+
| `--version` / `-v` | — | Print version and exit |
|
|
144
|
+
| `--help` / `-h` | — | Show help and exit |
|
|
145
|
+
|
|
146
|
+
### Exit Codes
|
|
147
|
+
|
|
148
|
+
| Code | Meaning |
|
|
149
|
+
|------|---------|
|
|
150
|
+
| `0` | All records valid |
|
|
151
|
+
| `1` | One or more records invalid |
|
|
152
|
+
| `2` | Operational error (file not found, bad arguments) |
|
|
153
|
+
|
|
154
|
+
### Human-readable output format
|
|
155
|
+
|
|
156
|
+
```
|
|
157
|
+
✓ events.jsonl: valid — 1000 line(s) checked, 0 errors
|
|
158
|
+
```
|
|
159
|
+
|
|
160
|
+
```
|
|
161
|
+
✗ events.jsonl: INVALID — 3 error(s) in 1000 line(s)
|
|
162
|
+
Line 42: [INVALID_JSON] JSON parse error: Unexpected end of JSON input
|
|
163
|
+
Line 87: [MISSING_KEY] Required key "id" is missing [key: id]
|
|
164
|
+
Line 99: [NOT_AN_OBJECT] Expected a JSON object but got array
|
|
165
|
+
```
|
|
166
|
+
|
|
167
|
+
### JSON output format
|
|
168
|
+
|
|
169
|
+
```json
|
|
170
|
+
{
|
|
171
|
+
"source": "events.jsonl",
|
|
172
|
+
"valid": false,
|
|
173
|
+
"totalLines": 1000,
|
|
174
|
+
"blankLines": 0,
|
|
175
|
+
"errorCount": 3,
|
|
176
|
+
"errors": [
|
|
177
|
+
{
|
|
178
|
+
"line": 42,
|
|
179
|
+
"type": "INVALID_JSON",
|
|
180
|
+
"message": "JSON parse error: Unexpected end of JSON input"
|
|
181
|
+
}
|
|
182
|
+
],
|
|
183
|
+
"truncated": false
|
|
184
|
+
}
|
|
185
|
+
```
|
|
@@ -0,0 +1,128 @@
|
|
|
1
|
+
# Design Decisions
|
|
2
|
+
|
|
3
|
+
This document records the key technical decisions made during the design of `jsonl-audit-kit` and the rationale behind each choice.
|
|
4
|
+
|
|
5
|
+
---
|
|
6
|
+
|
|
7
|
+
## 1. Pure JavaScript, No Runtime Dependencies
|
|
8
|
+
|
|
9
|
+
**Decision:** Use only Node.js built-in modules. Zero runtime dependencies.
|
|
10
|
+
|
|
11
|
+
**Rationale:**
|
|
12
|
+
- Zero attack surface from supply-chain vulnerabilities.
|
|
13
|
+
- No `node_modules` for the production install.
|
|
14
|
+
- Simpler for learners to read and understand.
|
|
15
|
+
- Node.js built-ins (`readline`, `fs`, `stream`) are well-maintained and stable.
|
|
16
|
+
|
|
17
|
+
**Trade-off:** External libraries like `@streamparser/json` offer more streaming flexibility, but the built-in `readline` interface with `for await...of` is sufficient for line-delimited formats and is conceptually simpler.
|
|
18
|
+
|
|
19
|
+
---
|
|
20
|
+
|
|
21
|
+
## 2. `readline.createInterface` for Line Splitting
|
|
22
|
+
|
|
23
|
+
**Decision:** Use `node:readline` with `crlfDelay: Infinity` for line splitting.
|
|
24
|
+
|
|
25
|
+
**Rationale:**
|
|
26
|
+
- Correctly handles both `\n` (Unix) and `\r\n` (Windows) line endings.
|
|
27
|
+
- The `for await...of` iteration pattern applies natural backpressure.
|
|
28
|
+
- Lines are delivered one at a time — no manual buffer accumulation.
|
|
29
|
+
- Well-documented, stable API available since Node.js 0.1.
|
|
30
|
+
|
|
31
|
+
**Memory implication:** `readline` buffers the current line. For very long lines (e.g. a 10 MB record), that line's bytes will be in memory simultaneously. This is unavoidable for line-based processing and is documented in the README.
|
|
32
|
+
|
|
33
|
+
---
|
|
34
|
+
|
|
35
|
+
## 3. ESM (ECMAScript Modules)
|
|
36
|
+
|
|
37
|
+
**Decision:** Use `"type": "module"` and native ESM throughout.
|
|
38
|
+
|
|
39
|
+
**Rationale:**
|
|
40
|
+
- Node.js 20+ has stable, full ESM support.
|
|
41
|
+
- ESM is the modern standard; avoids `require()` and CJS interop complexity.
|
|
42
|
+
- `import`/`export` syntax is clearer for learners.
|
|
43
|
+
- `node:test` supports ESM without additional configuration.
|
|
44
|
+
|
|
45
|
+
**Trade-off:** CommonJS consumers must use dynamic `import()`. This is documented in the README.
|
|
46
|
+
|
|
47
|
+
---
|
|
48
|
+
|
|
49
|
+
## 4. Bounded Diagnostic Storage
|
|
50
|
+
|
|
51
|
+
**Decision:** Store at most `maxErrors` diagnostics. Continue scanning the entire file. Count all errors separately.
|
|
52
|
+
|
|
53
|
+
**Rationale:**
|
|
54
|
+
- Prevents unbounded memory growth when validating enormous files with many errors.
|
|
55
|
+
- Developers still learn the total error count without loading all errors into memory.
|
|
56
|
+
- `truncated: true` signals when the stored list is incomplete.
|
|
57
|
+
- The default of 100 is generous for typical use but safe for large files.
|
|
58
|
+
|
|
59
|
+
**Alternative considered:** Streaming diagnostics via callback or async generator. Rejected for the MVP because it complicates the API without a strong MVP need. Marked as a future enhancement.
|
|
60
|
+
|
|
61
|
+
---
|
|
62
|
+
|
|
63
|
+
## 5. `auditFile` checks file existence with `stat` before streaming
|
|
64
|
+
|
|
65
|
+
**Decision:** Call `fs.stat` before opening the stream to check existence and type.
|
|
66
|
+
|
|
67
|
+
**Rationale:**
|
|
68
|
+
- Surfaces a clear, actionable error message (`Cannot access file "..."`) rather than the raw stream error.
|
|
69
|
+
- Distinguishes `ENOENT` (not found), `EACCES` (permission denied), and `EISDIR` (is a directory) early.
|
|
70
|
+
- Makes the API easier to use for callers who want to handle errors differently.
|
|
71
|
+
|
|
72
|
+
---
|
|
73
|
+
|
|
74
|
+
## 6. Separate `validateLine` function
|
|
75
|
+
|
|
76
|
+
**Decision:** Export `validateLine` as a named export from `validator.js`.
|
|
77
|
+
|
|
78
|
+
**Rationale:**
|
|
79
|
+
- Enables direct unit testing of the per-line validation logic without file I/O.
|
|
80
|
+
- Keeps the streaming orchestration in `auditStream` separate from the rule engine.
|
|
81
|
+
- Useful for advanced consumers who want to validate individual lines.
|
|
82
|
+
|
|
83
|
+
---
|
|
84
|
+
|
|
85
|
+
## 7. `requiredKeys` only checks plain objects
|
|
86
|
+
|
|
87
|
+
**Decision:** The required-key check is silently skipped when the parsed value is not a plain object (e.g. array, string, number).
|
|
88
|
+
|
|
89
|
+
**Rationale:**
|
|
90
|
+
- Required keys are only meaningful on objects. Applying them to arrays would produce confusing error messages.
|
|
91
|
+
- If the caller also sets `requireObject: true`, non-object records are reported via `NOT_AN_OBJECT` before the key check runs.
|
|
92
|
+
- If `requireObject` is false, the caller has opted out of structural enforcement and the key check would be nonsensical.
|
|
93
|
+
|
|
94
|
+
---
|
|
95
|
+
|
|
96
|
+
## 8. Node.js 20+ minimum
|
|
97
|
+
|
|
98
|
+
**Decision:** Set `engines.node` to `>=20.0.0`.
|
|
99
|
+
|
|
100
|
+
**Rationale:**
|
|
101
|
+
- Node.js 20 is the oldest Active LTS version at the time of development.
|
|
102
|
+
- `node:test` (the built-in test runner) became stable in Node.js 20.
|
|
103
|
+
- `for await...of` on readline is fully supported since Node.js 11.4.
|
|
104
|
+
|
|
105
|
+
---
|
|
106
|
+
|
|
107
|
+
## 9. Exit codes 0, 1, 2
|
|
108
|
+
|
|
109
|
+
**Decision:** CLI exits with 0 (valid), 1 (invalid records), 2 (operational error).
|
|
110
|
+
|
|
111
|
+
**Rationale:**
|
|
112
|
+
- Exit code 0 meaning "success" and 1 meaning "failure" is universal Unix convention.
|
|
113
|
+
- Using exit code 2 for operational errors (not found, bad arguments) distinguishes them from validation failures.
|
|
114
|
+
- This makes shell scripting and CI pipeline integration straightforward:
|
|
115
|
+
```sh
|
|
116
|
+
jsonl-audit events.jsonl || echo "Validation failed"
|
|
117
|
+
```
|
|
118
|
+
|
|
119
|
+
---
|
|
120
|
+
|
|
121
|
+
## 10. `--json` outputs to stdout, errors to stderr
|
|
122
|
+
|
|
123
|
+
**Decision:** JSON output (and human "valid" messages) go to stdout. Human-readable error listings go to stderr.
|
|
124
|
+
|
|
125
|
+
**Rationale:**
|
|
126
|
+
- Allows piping JSON output to `jq` or other tools without mixing error messages.
|
|
127
|
+
- Follows Unix conventions: informational/data output on stdout, diagnostics on stderr.
|
|
128
|
+
- In `--json` mode, **all** output is on stdout (the JSON object includes errors), so stderr is clean.
|