jsonl-audit-kit 0.0.0-stage → 0.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,259 @@
1
+ #!/usr/bin/env node
2
+ /**
3
+ * jsonl-audit.js — Command-line interface for jsonl-audit-kit
4
+ *
5
+ * Usage:
6
+ * jsonl-audit <file> Audit a JSONL file
7
+ * jsonl-audit - Read from stdin
8
+ * cat data.jsonl | jsonl-audit - Pipe data in
9
+ *
10
+ * Options:
11
+ * --require-object Require each record to be a JSON object
12
+ * --require-array Require each record to be a JSON array
13
+ * --required <keys> Comma-separated list of required top-level keys
14
+ * --report-blank-lines Treat blank lines as errors
15
+ * --max-errors <n> Maximum number of errors to store (default: 100)
16
+ * --max-line-bytes <n> Reject lines longer than this many bytes
17
+ * --json Output results as a JSON object
18
+ * --version Print version and exit
19
+ * --help Show usage and exit
20
+ *
21
+ * Exit codes:
22
+ * 0 File is valid (all records passed)
23
+ * 1 File is invalid (one or more records failed)
24
+ * 2 Operational error (file not found, bad arguments, etc.)
25
+ */
26
+
27
+ import { readFileSync } from 'node:fs';
28
+ import { createRequire } from 'node:module';
29
+ import { resolve, dirname } from 'node:path';
30
+ import { fileURLToPath } from 'node:url';
31
+ import { auditFile, auditStream } from '../src/index.js';
32
+
33
+ const __dirname = dirname(fileURLToPath(import.meta.url));
34
+
35
+ // ── Load package.json for --version ─────────────────────────────────────────
36
+ let pkg;
37
+ try {
38
+ const pkgPath = resolve(__dirname, '../package.json');
39
+ pkg = JSON.parse(readFileSync(pkgPath, 'utf8'));
40
+ } catch {
41
+ pkg = { version: '0.0.0' };
42
+ }
43
+
44
+ // ── Argument parsing ─────────────────────────────────────────────────────────
45
+
46
+ const EXIT_VALID = 0;
47
+ const EXIT_INVALID = 1;
48
+ const EXIT_ERROR = 2;
49
+
50
+ function showHelp() {
51
+ process.stdout.write(`
52
+ jsonl-audit — Audit JSONL (JSON Lines) files for invalid records
53
+
54
+ USAGE
55
+ jsonl-audit <file> Audit a file
56
+ jsonl-audit - Read from stdin
57
+ cat data.jsonl | jsonl-audit - Pipe data in
58
+
59
+ OPTIONS
60
+ --require-object Require each record to be a JSON object ({...})
61
+ --require-array Require each record to be a JSON array ([...])
62
+ --required <k1,k2,...> Comma-separated required top-level key names
63
+ --report-blank-lines Treat blank lines as validation errors
64
+ --max-errors <n> Max stored errors (default: 100). Scanning always
65
+ continues to completion; excess errors are counted
66
+ but not stored.
67
+ --max-line-bytes <n> Flag lines longer than n bytes
68
+ --json Output results as JSON (machine-readable)
69
+ --version Print version and exit
70
+ --help Show this help and exit
71
+
72
+ EXIT CODES
73
+ 0 All records are valid
74
+ 1 One or more records are invalid
75
+ 2 Operational error (file not found, permission denied, bad arguments)
76
+
77
+ EXAMPLES
78
+ jsonl-audit events.jsonl
79
+ jsonl-audit events.jsonl --require-object --required id,event
80
+ jsonl-audit events.jsonl --max-errors 10 --json
81
+ cat events.jsonl | jsonl-audit -
82
+ `.trimStart());
83
+ }
84
+
85
+ /**
86
+ * Parse argv into a structured object.
87
+ * Returns { target, options, jsonOutput, error }
88
+ */
89
+ function parseArgs(argv) {
90
+ const args = argv.slice(2);
91
+ let target = null;
92
+ let jsonOutput = false;
93
+ const opts = {};
94
+ let error = null;
95
+
96
+ let i = 0;
97
+ while (i < args.length) {
98
+ const arg = args[i];
99
+
100
+ if (arg === '--help' || arg === '-h') {
101
+ showHelp();
102
+ process.exit(EXIT_VALID);
103
+ }
104
+
105
+ if (arg === '--version' || arg === '-v') {
106
+ process.stdout.write(`jsonl-audit-kit v${pkg.version}\n`);
107
+ process.exit(EXIT_VALID);
108
+ }
109
+
110
+ if (arg === '--json') {
111
+ jsonOutput = true;
112
+ i++;
113
+ continue;
114
+ }
115
+
116
+ if (arg === '--require-object') {
117
+ opts.requireObject = true;
118
+ i++;
119
+ continue;
120
+ }
121
+
122
+ if (arg === '--require-array') {
123
+ opts.requireArray = true;
124
+ i++;
125
+ continue;
126
+ }
127
+
128
+ if (arg === '--report-blank-lines') {
129
+ opts.reportBlankLines = true;
130
+ i++;
131
+ continue;
132
+ }
133
+
134
+ if (arg === '--required') {
135
+ const val = args[i + 1];
136
+ if (!val || val.startsWith('--')) {
137
+ error = '--required requires a comma-separated list of key names';
138
+ break;
139
+ }
140
+ opts.requiredKeys = val.split(',').map((k) => k.trim()).filter(Boolean);
141
+ i += 2;
142
+ continue;
143
+ }
144
+
145
+ if (arg === '--max-errors') {
146
+ const val = parseInt(args[i + 1], 10);
147
+ if (!Number.isInteger(val) || val < 1) {
148
+ error = '--max-errors requires a positive integer';
149
+ break;
150
+ }
151
+ opts.maxErrors = val;
152
+ i += 2;
153
+ continue;
154
+ }
155
+
156
+ if (arg === '--max-line-bytes') {
157
+ const val = parseInt(args[i + 1], 10);
158
+ if (!Number.isInteger(val) || val < 1) {
159
+ error = '--max-line-bytes requires a positive integer';
160
+ break;
161
+ }
162
+ opts.maxLineBytes = val;
163
+ i += 2;
164
+ continue;
165
+ }
166
+
167
+ if (arg.startsWith('--')) {
168
+ error = `Unknown option: ${arg}`;
169
+ break;
170
+ }
171
+
172
+ // Positional: target file or '-' for stdin
173
+ if (target !== null) {
174
+ error = 'Too many positional arguments — expected one file path or "-"';
175
+ break;
176
+ }
177
+ target = arg;
178
+ i++;
179
+ }
180
+
181
+ return { target, options: opts, jsonOutput, error };
182
+ }
183
+
184
+ // ── Output helpers ────────────────────────────────────────────────────────────
185
+
186
+ function humanOutput(result, source) {
187
+ const label = source === '-' ? '<stdin>' : source;
188
+
189
+ if (result.valid) {
190
+ process.stdout.write(
191
+ `✓ ${label}: valid — ${result.totalLines} line(s) checked, 0 errors\n`,
192
+ );
193
+ return;
194
+ }
195
+
196
+ process.stderr.write(
197
+ `✗ ${label}: INVALID — ${result.errorCount} error(s) in ${result.totalLines} line(s)\n`,
198
+ );
199
+
200
+ if (result.truncated) {
201
+ process.stderr.write(
202
+ ` (showing first ${result.errors.length} of ${result.errorCount} errors)\n`,
203
+ );
204
+ }
205
+
206
+ for (const diag of result.errors) {
207
+ const keyInfo = diag.key ? ` [key: ${diag.key}]` : '';
208
+ process.stderr.write(
209
+ ` Line ${diag.line}: [${diag.type}] ${diag.message}${keyInfo}\n`,
210
+ );
211
+ }
212
+ }
213
+
214
+ function jsonOutputFn(result, source) {
215
+ const output = {
216
+ source: source === '-' ? '<stdin>' : source,
217
+ ...result,
218
+ };
219
+ process.stdout.write(JSON.stringify(output, null, 2) + '\n');
220
+ }
221
+
222
+ // ── Main ──────────────────────────────────────────────────────────────────────
223
+
224
+ async function main() {
225
+ const { target, options, jsonOutput, error } = parseArgs(process.argv);
226
+
227
+ if (error) {
228
+ process.stderr.write(`Error: ${error}\n\nRun with --help for usage.\n`);
229
+ process.exit(EXIT_ERROR);
230
+ }
231
+
232
+ if (target === null) {
233
+ showHelp();
234
+ process.exit(EXIT_ERROR);
235
+ }
236
+
237
+ let result;
238
+ try {
239
+ if (target === '-') {
240
+ // Read from stdin
241
+ result = await auditStream(process.stdin, options);
242
+ } else {
243
+ result = await auditFile(target, options);
244
+ }
245
+ } catch (err) {
246
+ process.stderr.write(`Error: ${err.message}\n`);
247
+ process.exit(EXIT_ERROR);
248
+ }
249
+
250
+ if (jsonOutput) {
251
+ jsonOutputFn(result, target);
252
+ } else {
253
+ humanOutput(result, target);
254
+ }
255
+
256
+ process.exit(result.valid ? EXIT_VALID : EXIT_INVALID);
257
+ }
258
+
259
+ main();
package/docs/api.md ADDED
@@ -0,0 +1,185 @@
1
+ # API Reference — jsonl-audit-kit
2
+
3
+ ## Functions
4
+
5
+ ### `auditFile(filePath, options?)`
6
+
7
+ Audit a JSONL file at the given path.
8
+
9
+ ```js
10
+ import { auditFile } from 'jsonl-audit-kit';
11
+
12
+ const result = await auditFile('./events.jsonl', {
13
+ requireObject: true,
14
+ requiredKeys: ['id', 'event'],
15
+ maxErrors: 50,
16
+ });
17
+ ```
18
+
19
+ **Parameters**
20
+
21
+ | Parameter | Type | Required | Description |
22
+ |-----------|------|----------|-------------|
23
+ | `filePath` | `string` | Yes | Absolute or relative path to the JSONL file |
24
+ | `options` | `AuditOptions` | No | Validation options (see below) |
25
+
26
+ **Returns:** `Promise<AuditResult>`
27
+
28
+ **Throws:**
29
+ - `TypeError` — `filePath` is not a string, or options are invalid
30
+ - `Error` (code `ENOENT`) — file does not exist
31
+ - `Error` (code `EISDIR`) — path is a directory
32
+ - `Error` (code `EACCES`) — file is not readable
33
+
34
+ ---
35
+
36
+ ### `auditStream(stream, options?)`
37
+
38
+ Audit a readable Node.js stream containing JSONL data.
39
+
40
+ ```js
41
+ import { auditStream } from 'jsonl-audit-kit';
42
+ import { createReadStream } from 'node:fs';
43
+
44
+ const stream = createReadStream('./events.jsonl', { encoding: 'utf8' });
45
+ const result = await auditStream(stream, { requireObject: true });
46
+ ```
47
+
48
+ **Parameters**
49
+
50
+ | Parameter | Type | Required | Description |
51
+ |-----------|------|----------|-------------|
52
+ | `stream` | `Readable` | Yes | A readable Node.js stream |
53
+ | `options` | `AuditOptions` | No | Validation options (see below) |
54
+
55
+ **Returns:** `Promise<AuditResult>`
56
+
57
+ ---
58
+
59
+ ## Types
60
+
61
+ ### `AuditOptions`
62
+
63
+ | Property | Type | Default | Description |
64
+ |----------|------|---------|-------------|
65
+ | `requireObject` | `boolean` | `false` | Require every record to be a JSON object `{...}` |
66
+ | `requireArray` | `boolean` | `false` | Require every record to be a JSON array `[...]` |
67
+ | `requiredKeys` | `string[]` | `[]` | Keys that must be present in every object record |
68
+ | `reportBlankLines` | `boolean` | `false` | Treat blank lines as errors |
69
+ | `maxErrors` | `number` | `100` | Maximum stored diagnostics (scanning always completes) |
70
+ | `maxLineBytes` | `number \| null` | `null` | Flag lines longer than this many bytes |
71
+
72
+ `requireObject` and `requireArray` cannot both be `true`.
73
+
74
+ ---
75
+
76
+ ### `AuditResult`
77
+
78
+ | Property | Type | Description |
79
+ |----------|------|-------------|
80
+ | `valid` | `boolean` | `true` if no errors were found |
81
+ | `totalLines` | `number` | Total number of lines read (including blank lines) |
82
+ | `blankLines` | `number` | Number of blank / whitespace-only lines |
83
+ | `errorCount` | `number` | Total number of validation errors (may exceed `errors.length`) |
84
+ | `errors` | `Diagnostic[]` | Stored diagnostics (at most `maxErrors` entries) |
85
+ | `truncated` | `boolean` | `true` when `errorCount > errors.length` |
86
+
87
+ ---
88
+
89
+ ### `Diagnostic`
90
+
91
+ | Property | Type | Present when |
92
+ |----------|------|--------------|
93
+ | `line` | `number` | Always — 1-based line number |
94
+ | `type` | `DiagnosticTypeName` | Always — category of the problem |
95
+ | `message` | `string` | Always — human-readable description |
96
+ | `key` | `string` | `MISSING_KEY` diagnostics only |
97
+ | `length` | `number` | `LINE_TOO_LONG` diagnostics only |
98
+ | `max` | `number` | `LINE_TOO_LONG` diagnostics only |
99
+
100
+ ---
101
+
102
+ ### `DiagnosticType`
103
+
104
+ ```js
105
+ import { DiagnosticType } from 'jsonl-audit-kit';
106
+
107
+ DiagnosticType.INVALID_JSON // 'INVALID_JSON'
108
+ DiagnosticType.BLANK_LINE // 'BLANK_LINE'
109
+ DiagnosticType.NOT_AN_OBJECT // 'NOT_AN_OBJECT'
110
+ DiagnosticType.NOT_AN_ARRAY // 'NOT_AN_ARRAY'
111
+ DiagnosticType.MISSING_KEY // 'MISSING_KEY'
112
+ DiagnosticType.LINE_TOO_LONG // 'LINE_TOO_LONG'
113
+ ```
114
+
115
+ | Constant | Meaning |
116
+ |----------|---------|
117
+ | `INVALID_JSON` | Line is not valid JSON |
118
+ | `BLANK_LINE` | Line is blank (only when `reportBlankLines: true`) |
119
+ | `NOT_AN_OBJECT` | Parsed value is not a plain object (when `requireObject: true`) |
120
+ | `NOT_AN_ARRAY` | Parsed value is not an array (when `requireArray: true`) |
121
+ | `MISSING_KEY` | A required key is absent from the top-level object |
122
+ | `LINE_TOO_LONG` | Line exceeds `maxLineBytes` |
123
+
124
+ ---
125
+
126
+ ## CLI Reference
127
+
128
+ ```
129
+ jsonl-audit <file|-> [options]
130
+ ```
131
+
132
+ ### Options
133
+
134
+ | Flag | Argument | Description |
135
+ |------|----------|-------------|
136
+ | `--require-object` | — | Require every record to be a JSON object |
137
+ | `--require-array` | — | Require every record to be a JSON array |
138
+ | `--required` | `k1,k2,...` | Comma-separated required key names |
139
+ | `--report-blank-lines` | — | Treat blank lines as errors |
140
+ | `--max-errors` | `n` | Max stored errors (default: 100) |
141
+ | `--max-line-bytes` | `n` | Flag lines longer than n bytes |
142
+ | `--json` | — | Machine-readable JSON output to stdout |
143
+ | `--version` / `-v` | — | Print version and exit |
144
+ | `--help` / `-h` | — | Show help and exit |
145
+
146
+ ### Exit Codes
147
+
148
+ | Code | Meaning |
149
+ |------|---------|
150
+ | `0` | All records valid |
151
+ | `1` | One or more records invalid |
152
+ | `2` | Operational error (file not found, bad arguments) |
153
+
154
+ ### Human-readable output format
155
+
156
+ ```
157
+ ✓ events.jsonl: valid — 1000 line(s) checked, 0 errors
158
+ ```
159
+
160
+ ```
161
+ ✗ events.jsonl: INVALID — 3 error(s) in 1000 line(s)
162
+ Line 42: [INVALID_JSON] JSON parse error: Unexpected end of JSON input
163
+ Line 87: [MISSING_KEY] Required key "id" is missing [key: id]
164
+ Line 99: [NOT_AN_OBJECT] Expected a JSON object but got array
165
+ ```
166
+
167
+ ### JSON output format
168
+
169
+ ```json
170
+ {
171
+ "source": "events.jsonl",
172
+ "valid": false,
173
+ "totalLines": 1000,
174
+ "blankLines": 0,
175
+ "errorCount": 3,
176
+ "errors": [
177
+ {
178
+ "line": 42,
179
+ "type": "INVALID_JSON",
180
+ "message": "JSON parse error: Unexpected end of JSON input"
181
+ }
182
+ ],
183
+ "truncated": false
184
+ }
185
+ ```
@@ -0,0 +1,128 @@
1
+ # Design Decisions
2
+
3
+ This document records the key technical decisions made during the design of `jsonl-audit-kit` and the rationale behind each choice.
4
+
5
+ ---
6
+
7
+ ## 1. Pure JavaScript, No Runtime Dependencies
8
+
9
+ **Decision:** Use only Node.js built-in modules. Zero runtime dependencies.
10
+
11
+ **Rationale:**
12
+ - Zero attack surface from supply-chain vulnerabilities.
13
+ - No `node_modules` for the production install.
14
+ - Simpler for learners to read and understand.
15
+ - Node.js built-ins (`readline`, `fs`, `stream`) are well-maintained and stable.
16
+
17
+ **Trade-off:** External libraries like `@streamparser/json` offer more streaming flexibility, but the built-in `readline` interface with `for await...of` is sufficient for line-delimited formats and is conceptually simpler.
18
+
19
+ ---
20
+
21
+ ## 2. `readline.createInterface` for Line Splitting
22
+
23
+ **Decision:** Use `node:readline` with `crlfDelay: Infinity` for line splitting.
24
+
25
+ **Rationale:**
26
+ - Correctly handles both `\n` (Unix) and `\r\n` (Windows) line endings.
27
+ - The `for await...of` iteration pattern applies natural backpressure.
28
+ - Lines are delivered one at a time — no manual buffer accumulation.
29
+ - Well-documented, stable API available since Node.js 0.1.
30
+
31
+ **Memory implication:** `readline` buffers the current line. For very long lines (e.g. a 10 MB record), that line's bytes will be in memory simultaneously. This is unavoidable for line-based processing and is documented in the README.
32
+
33
+ ---
34
+
35
+ ## 3. ESM (ECMAScript Modules)
36
+
37
+ **Decision:** Use `"type": "module"` and native ESM throughout.
38
+
39
+ **Rationale:**
40
+ - Node.js 20+ has stable, full ESM support.
41
+ - ESM is the modern standard; avoids `require()` and CJS interop complexity.
42
+ - `import`/`export` syntax is clearer for learners.
43
+ - `node:test` supports ESM without additional configuration.
44
+
45
+ **Trade-off:** CommonJS consumers must use dynamic `import()`. This is documented in the README.
46
+
47
+ ---
48
+
49
+ ## 4. Bounded Diagnostic Storage
50
+
51
+ **Decision:** Store at most `maxErrors` diagnostics. Continue scanning the entire file. Count all errors separately.
52
+
53
+ **Rationale:**
54
+ - Prevents unbounded memory growth when validating enormous files with many errors.
55
+ - Developers still learn the total error count without loading all errors into memory.
56
+ - `truncated: true` signals when the stored list is incomplete.
57
+ - The default of 100 is generous for typical use but safe for large files.
58
+
59
+ **Alternative considered:** Streaming diagnostics via callback or async generator. Rejected for the MVP because it complicates the API without a strong MVP need. Marked as a future enhancement.
60
+
61
+ ---
62
+
63
+ ## 5. `auditFile` checks file existence with `stat` before streaming
64
+
65
+ **Decision:** Call `fs.stat` before opening the stream to check existence and type.
66
+
67
+ **Rationale:**
68
+ - Surfaces a clear, actionable error message (`Cannot access file "..."`) rather than the raw stream error.
69
+ - Distinguishes `ENOENT` (not found), `EACCES` (permission denied), and `EISDIR` (is a directory) early.
70
+ - Makes the API easier to use for callers who want to handle errors differently.
71
+
72
+ ---
73
+
74
+ ## 6. Separate `validateLine` function
75
+
76
+ **Decision:** Export `validateLine` as a named export from `validator.js`.
77
+
78
+ **Rationale:**
79
+ - Enables direct unit testing of the per-line validation logic without file I/O.
80
+ - Keeps the streaming orchestration in `auditStream` separate from the rule engine.
81
+ - Useful for advanced consumers who want to validate individual lines.
82
+
83
+ ---
84
+
85
+ ## 7. `requiredKeys` only checks plain objects
86
+
87
+ **Decision:** The required-key check is silently skipped when the parsed value is not a plain object (e.g. array, string, number).
88
+
89
+ **Rationale:**
90
+ - Required keys are only meaningful on objects. Applying them to arrays would produce confusing error messages.
91
+ - If the caller also sets `requireObject: true`, non-object records are reported via `NOT_AN_OBJECT` before the key check runs.
92
+ - If `requireObject` is false, the caller has opted out of structural enforcement and the key check would be nonsensical.
93
+
94
+ ---
95
+
96
+ ## 8. Node.js 20+ minimum
97
+
98
+ **Decision:** Set `engines.node` to `>=20.0.0`.
99
+
100
+ **Rationale:**
101
+ - Node.js 20 is the oldest Active LTS version at the time of development.
102
+ - `node:test` (the built-in test runner) became stable in Node.js 20.
103
+ - `for await...of` on readline is fully supported since Node.js 11.4.
104
+
105
+ ---
106
+
107
+ ## 9. Exit codes 0, 1, 2
108
+
109
+ **Decision:** CLI exits with 0 (valid), 1 (invalid records), 2 (operational error).
110
+
111
+ **Rationale:**
112
+ - Exit code 0 meaning "success" and 1 meaning "failure" is universal Unix convention.
113
+ - Using exit code 2 for operational errors (not found, bad arguments) distinguishes them from validation failures.
114
+ - This makes shell scripting and CI pipeline integration straightforward:
115
+ ```sh
116
+ jsonl-audit events.jsonl || echo "Validation failed"
117
+ ```
118
+
119
+ ---
120
+
121
+ ## 10. `--json` outputs to stdout, errors to stderr
122
+
123
+ **Decision:** JSON output (and human "valid" messages) go to stdout. Human-readable error listings go to stderr.
124
+
125
+ **Rationale:**
126
+ - Allows piping JSON output to `jq` or other tools without mixing error messages.
127
+ - Follows Unix conventions: informational/data output on stdout, diagnostics on stderr.
128
+ - In `--json` mode, **all** output is on stdout (the JSON object includes errors), so stderr is clean.