jsonl-audit-kit 0.0.0-stage → 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +42 -0
- package/CONTRIBUTING.md +71 -0
- package/LICENSE +21 -0
- package/README.md +364 -2
- package/bin/jsonl-audit.js +259 -0
- package/docs/api.md +185 -0
- package/docs/design-decisions.md +128 -0
- package/docs/research.md +176 -0
- package/examples/invalid.jsonl +5 -0
- package/examples/usage.js +94 -0
- package/examples/valid.jsonl +5 -0
- package/package.json +60 -4
- package/src/diagnostics.js +124 -0
- package/src/index.js +99 -0
- package/src/utils/validate-options.js +68 -0
- package/src/validator.js +153 -0
- package/types/index.d.ts +195 -0
package/docs/research.md
ADDED
|
@@ -0,0 +1,176 @@
|
|
|
1
|
+
# Research: JSONL Ecosystem and Competitors
|
|
2
|
+
|
|
3
|
+
> This document records findings from pre-implementation research.
|
|
4
|
+
> All data was gathered before the implementation was written.
|
|
5
|
+
> Package statistics (downloads, stars) are **not included** because
|
|
6
|
+
> they fluctuate and were not verified at research time.
|
|
7
|
+
|
|
8
|
+
---
|
|
9
|
+
|
|
10
|
+
## 1. Package Name Availability
|
|
11
|
+
|
|
12
|
+
**Name checked:** `jsonl-audit-kit`
|
|
13
|
+
|
|
14
|
+
**Finding:** No package named `jsonl-audit-kit` was found on the npm registry
|
|
15
|
+
at research time. The name appears to be available.
|
|
16
|
+
|
|
17
|
+
**Note:** Availability can change between research and publication.
|
|
18
|
+
Always run `npm view jsonl-audit-kit` immediately before publishing.
|
|
19
|
+
|
|
20
|
+
---
|
|
21
|
+
|
|
22
|
+
## 2. Existing Ecosystem Packages
|
|
23
|
+
|
|
24
|
+
### 2.1 `ndjson`
|
|
25
|
+
|
|
26
|
+
| Attribute | Detail |
|
|
27
|
+
|-----------|--------|
|
|
28
|
+
| Purpose | Parse and stringify NDJSON/JSONL streams |
|
|
29
|
+
| Mechanism | Node.js Transform streams |
|
|
30
|
+
| CLI | No dedicated CLI |
|
|
31
|
+
| Validation | None — it is a parser/serialiser only |
|
|
32
|
+
| Maintenance | Maintenance has slowed; older codebase |
|
|
33
|
+
| License | BSD-2-Clause |
|
|
34
|
+
|
|
35
|
+
**Limitations relevant to `jsonl-audit-kit`:**
|
|
36
|
+
- No validation, no diagnostics, no line-level error reporting.
|
|
37
|
+
- Not designed for auditing — it fails or silently skips on bad input.
|
|
38
|
+
- No CLI for quick file inspection.
|
|
39
|
+
|
|
40
|
+
---
|
|
41
|
+
|
|
42
|
+
### 2.2 `stream-json`
|
|
43
|
+
|
|
44
|
+
| Attribute | Detail |
|
|
45
|
+
|-----------|--------|
|
|
46
|
+
| Purpose | Token-stream parsing of large JSON documents |
|
|
47
|
+
| Mechanism | Modular pipeline with `pick`, `ignore`, `filter` |
|
|
48
|
+
| CLI | No |
|
|
49
|
+
| Validation | None built-in; pairs with AJV |
|
|
50
|
+
| Maintenance | Actively maintained; supports ESM, TypeScript |
|
|
51
|
+
| License | BSD-3-Clause |
|
|
52
|
+
|
|
53
|
+
**Limitations:**
|
|
54
|
+
- Designed for single large JSON files, not line-delimited files.
|
|
55
|
+
- Steep learning curve for simple use cases.
|
|
56
|
+
- No concept of "per-line audit" with diagnostics.
|
|
57
|
+
|
|
58
|
+
---
|
|
59
|
+
|
|
60
|
+
### 2.3 `JSONStream` (deprecated in practice)
|
|
61
|
+
|
|
62
|
+
| Attribute | Detail |
|
|
63
|
+
|-----------|--------|
|
|
64
|
+
| Purpose | JSONPath-based streaming of large arrays/objects |
|
|
65
|
+
| Mechanism | CouchDB-style JSONPath expressions |
|
|
66
|
+
| CLI | No |
|
|
67
|
+
| Validation | None |
|
|
68
|
+
| Maintenance | Effectively abandoned; relies on `event-stream` |
|
|
69
|
+
| License | Apache-2.0 / MIT |
|
|
70
|
+
|
|
71
|
+
**Not recommended** for new projects due to maintenance and security history.
|
|
72
|
+
|
|
73
|
+
---
|
|
74
|
+
|
|
75
|
+
### 2.4 `@streamparser/json`
|
|
76
|
+
|
|
77
|
+
| Attribute | Detail |
|
|
78
|
+
|-----------|--------|
|
|
79
|
+
| Purpose | Dependency-free streaming JSON parser |
|
|
80
|
+
| Mechanism | WHATWG TransformStream / Node.js streams |
|
|
81
|
+
| CLI | No |
|
|
82
|
+
| Validation | None built-in |
|
|
83
|
+
| Maintenance | Actively maintained; cross-runtime |
|
|
84
|
+
| License | MIT |
|
|
85
|
+
|
|
86
|
+
**Limitations:**
|
|
87
|
+
- No line-level error diagnostics.
|
|
88
|
+
- No CLI for file auditing.
|
|
89
|
+
- Not designed as a file-audit tool.
|
|
90
|
+
|
|
91
|
+
---
|
|
92
|
+
|
|
93
|
+
### 2.5 `ajv` (Another JSON Validator)
|
|
94
|
+
|
|
95
|
+
| Attribute | Detail |
|
|
96
|
+
|-----------|--------|
|
|
97
|
+
| Purpose | JSON Schema validation |
|
|
98
|
+
| Mechanism | Compiled validators from JSON Schema |
|
|
99
|
+
| CLI | Separate `ajv-cli` package |
|
|
100
|
+
| Validation | Full JSON Schema draft 2020-12 support |
|
|
101
|
+
| Maintenance | Actively maintained; widely used |
|
|
102
|
+
| License | MIT |
|
|
103
|
+
|
|
104
|
+
**Relationship to `jsonl-audit-kit`:**
|
|
105
|
+
AJV validates individual objects against a schema. It does not read JSONL
|
|
106
|
+
files, does not stream, and does not provide line-level diagnostics. A
|
|
107
|
+
user wanting both would need to write glue code. `jsonl-audit-kit` is
|
|
108
|
+
complementary rather than competing.
|
|
109
|
+
|
|
110
|
+
---
|
|
111
|
+
|
|
112
|
+
### 2.6 `@hexmon_tech/audit-sink-file-jsonl`
|
|
113
|
+
|
|
114
|
+
| Attribute | Detail |
|
|
115
|
+
|-----------|--------|
|
|
116
|
+
| Purpose | Write audit log events to a JSONL file (output sink) |
|
|
117
|
+
| Maintenance | Small / niche |
|
|
118
|
+
|
|
119
|
+
**Not relevant** — this writes JSONL; it does not validate it.
|
|
120
|
+
|
|
121
|
+
---
|
|
122
|
+
|
|
123
|
+
## 3. Identified Gap
|
|
124
|
+
|
|
125
|
+
None of the researched packages provides all of:
|
|
126
|
+
|
|
127
|
+
1. **Streaming read** of a JSONL file without loading it into memory.
|
|
128
|
+
2. **Per-line validation** with line numbers preserved.
|
|
129
|
+
3. **Actionable diagnostics** (what went wrong, on which line, and why).
|
|
130
|
+
4. **Configurable validation rules** (require objects, required keys, size limits).
|
|
131
|
+
5. **Bounded error accumulation** (collect N errors, count all, avoid OOM).
|
|
132
|
+
6. **Developer-friendly CLI** with human and machine-readable output.
|
|
133
|
+
7. **Correct exit codes** for CI pipeline integration.
|
|
134
|
+
8. **Zero runtime dependencies** using only Node.js built-ins.
|
|
135
|
+
|
|
136
|
+
This is the gap `jsonl-audit-kit` fills.
|
|
137
|
+
|
|
138
|
+
---
|
|
139
|
+
|
|
140
|
+
## 4. Differentiation Summary
|
|
141
|
+
|
|
142
|
+
| Feature | ndjson | stream-json | @streamparser/json | **jsonl-audit-kit** |
|
|
143
|
+
|---------|--------|-------------|---------------------|----------------------|
|
|
144
|
+
| Streaming read | ✓ | ✓ | ✓ | ✓ |
|
|
145
|
+
| Per-line diagnostics | ✗ | ✗ | ✗ | ✓ |
|
|
146
|
+
| Line numbering | ✗ | ✗ | ✗ | ✓ |
|
|
147
|
+
| Configurable rules | ✗ | ✗ | ✗ | ✓ |
|
|
148
|
+
| Bounded error list | ✗ | ✗ | ✗ | ✓ |
|
|
149
|
+
| CLI with exit codes | ✗ | ✗ | ✗ | ✓ |
|
|
150
|
+
| JSON output mode | ✗ | ✗ | ✗ | ✓ |
|
|
151
|
+
| Zero runtime deps | ✓ | ✗ | ✓ | ✓ |
|
|
152
|
+
| Node.js built-ins only | ✓ | ✗ | ✗ | ✓ |
|
|
153
|
+
|
|
154
|
+
---
|
|
155
|
+
|
|
156
|
+
## 5. Node.js API Research
|
|
157
|
+
|
|
158
|
+
The implementation uses only Node.js built-in modules:
|
|
159
|
+
|
|
160
|
+
- `node:fs` — `createReadStream`, `stat`
|
|
161
|
+
- `node:fs/promises` — `stat`
|
|
162
|
+
- `node:readline` — `createInterface` with `for await ... of` async iteration
|
|
163
|
+
- `node:stream` — `Readable`
|
|
164
|
+
- `node:path` — `resolve`, `join`, `dirname`
|
|
165
|
+
- `node:process` — `process.stdin`, `process.stdout`, `process.stderr`, `process.exit`
|
|
166
|
+
- `node:module` — `createRequire` (not used in final impl)
|
|
167
|
+
- `node:url` — `fileURLToPath`
|
|
168
|
+
- `node:child_process` — `spawn` (test-only)
|
|
169
|
+
- `node:test` — test runner (test-only)
|
|
170
|
+
- `node:assert` — assertions (test-only)
|
|
171
|
+
|
|
172
|
+
`readline.createInterface` with `crlfDelay: Infinity` correctly handles both
|
|
173
|
+
`\n` and `\r\n` line endings without additional processing.
|
|
174
|
+
|
|
175
|
+
The `for await ... of` loop on the readline interface is the idiomatic,
|
|
176
|
+
backpressure-aware way to read a stream line-by-line in Node.js 12+.
|
|
@@ -0,0 +1,94 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* usage.js — API usage examples for jsonl-audit-kit
|
|
3
|
+
*
|
|
4
|
+
* Run with: node examples/usage.js
|
|
5
|
+
*/
|
|
6
|
+
|
|
7
|
+
import { auditFile, auditStream, DiagnosticType } from '../src/index.js';
|
|
8
|
+
import { Readable } from 'node:stream';
|
|
9
|
+
import { fileURLToPath } from 'node:url';
|
|
10
|
+
import { dirname, join } from 'node:path';
|
|
11
|
+
|
|
12
|
+
const __dirname = dirname(fileURLToPath(import.meta.url));
|
|
13
|
+
|
|
14
|
+
// ── Example 1: Audit a valid file ─────────────────────────────────────────────
|
|
15
|
+
|
|
16
|
+
console.log('=== Example 1: Valid file ===');
|
|
17
|
+
{
|
|
18
|
+
const result = await auditFile(join(__dirname, 'valid.jsonl'));
|
|
19
|
+
console.log('valid:', result.valid);
|
|
20
|
+
console.log('totalLines:', result.totalLines);
|
|
21
|
+
console.log('errorCount:', result.errorCount);
|
|
22
|
+
// → valid: true, totalLines: 5, errorCount: 0
|
|
23
|
+
}
|
|
24
|
+
|
|
25
|
+
// ── Example 2: Audit a file with errors ──────────────────────────────────────
|
|
26
|
+
|
|
27
|
+
console.log('\n=== Example 2: File with errors ===');
|
|
28
|
+
{
|
|
29
|
+
const result = await auditFile(join(__dirname, 'invalid.jsonl'), {
|
|
30
|
+
requireObject: true,
|
|
31
|
+
requiredKeys: ['id', 'event'],
|
|
32
|
+
});
|
|
33
|
+
|
|
34
|
+
console.log('valid:', result.valid);
|
|
35
|
+
console.log('totalLines:', result.totalLines);
|
|
36
|
+
console.log('errorCount:', result.errorCount);
|
|
37
|
+
for (const err of result.errors) {
|
|
38
|
+
const key = err.key ? ` [key: ${err.key}]` : '';
|
|
39
|
+
console.log(` Line ${err.line}: [${err.type}] ${err.message}${key}`);
|
|
40
|
+
}
|
|
41
|
+
}
|
|
42
|
+
|
|
43
|
+
// ── Example 3: Audit a stream ────────────────────────────────────────────────
|
|
44
|
+
|
|
45
|
+
console.log('\n=== Example 3: In-memory stream ===');
|
|
46
|
+
{
|
|
47
|
+
const data = [
|
|
48
|
+
'{"id":1,"event":"created"}',
|
|
49
|
+
'{"id":2,"event":"updated"}',
|
|
50
|
+
'this is not json',
|
|
51
|
+
'{"id":4,"event":"deleted"}',
|
|
52
|
+
].join('\n') + '\n';
|
|
53
|
+
|
|
54
|
+
const stream = Readable.from([data]);
|
|
55
|
+
const result = await auditStream(stream, { requireObject: true });
|
|
56
|
+
|
|
57
|
+
console.log('valid:', result.valid);
|
|
58
|
+
console.log('errorCount:', result.errorCount);
|
|
59
|
+
for (const err of result.errors) {
|
|
60
|
+
console.log(` Line ${err.line}: ${err.message}`);
|
|
61
|
+
}
|
|
62
|
+
}
|
|
63
|
+
|
|
64
|
+
// ── Example 4: Bounded error collection ──────────────────────────────────────
|
|
65
|
+
|
|
66
|
+
console.log('\n=== Example 4: Bounded error collection ===');
|
|
67
|
+
{
|
|
68
|
+
const lines = Array.from({ length: 20 }, (_, i) => `BAD LINE ${i}`).join('\n') + '\n';
|
|
69
|
+
const stream = Readable.from([lines]);
|
|
70
|
+
|
|
71
|
+
const result = await auditStream(stream, { maxErrors: 5 });
|
|
72
|
+
|
|
73
|
+
console.log('totalLines:', result.totalLines);
|
|
74
|
+
console.log('errorCount:', result.errorCount); // 20
|
|
75
|
+
console.log('stored errors:', result.errors.length); // 5 (capped)
|
|
76
|
+
console.log('truncated:', result.truncated); // true
|
|
77
|
+
}
|
|
78
|
+
|
|
79
|
+
// ── Example 5: Using DiagnosticType for filtering ────────────────────────────
|
|
80
|
+
|
|
81
|
+
console.log('\n=== Example 5: Filtering by diagnostic type ===');
|
|
82
|
+
{
|
|
83
|
+
const data = '{"id":1}\n[1,2]\n{"id":3}\nBAD\n';
|
|
84
|
+
const stream = Readable.from([data]);
|
|
85
|
+
const result = await auditStream(stream, { requireObject: true });
|
|
86
|
+
|
|
87
|
+
const parseErrors = result.errors.filter((e) => e.type === DiagnosticType.INVALID_JSON);
|
|
88
|
+
const typeErrors = result.errors.filter((e) => e.type === DiagnosticType.NOT_AN_OBJECT);
|
|
89
|
+
|
|
90
|
+
console.log(`INVALID_JSON errors: ${parseErrors.length}`);
|
|
91
|
+
console.log(`NOT_AN_OBJECT errors: ${typeErrors.length}`);
|
|
92
|
+
}
|
|
93
|
+
|
|
94
|
+
console.log('\nAll examples complete.');
|
|
@@ -0,0 +1,5 @@
|
|
|
1
|
+
{"id":1,"event":"created","ts":"2024-01-15T08:00:00Z","user":"alice"}
|
|
2
|
+
{"id":2,"event":"updated","ts":"2024-01-15T08:05:00Z","user":"bob"}
|
|
3
|
+
{"id":3,"event":"deleted","ts":"2024-01-15T08:10:00Z","user":"alice"}
|
|
4
|
+
{"id":4,"event":"created","ts":"2024-01-15T08:15:00Z","user":"carol"}
|
|
5
|
+
{"id":5,"event":"viewed","ts":"2024-01-15T08:20:00Z","user":"bob"}
|
package/package.json
CHANGED
|
@@ -1,6 +1,62 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "jsonl-audit-kit",
|
|
3
|
-
"version": "0.
|
|
4
|
-
"
|
|
5
|
-
"
|
|
6
|
-
|
|
3
|
+
"version": "0.1.0",
|
|
4
|
+
"description": "Audit JSONL (JSON Lines / NDJSON) files and identify invalid records with actionable line-level diagnostics. Streaming, zero-dependency, CI-friendly.",
|
|
5
|
+
"keywords": [
|
|
6
|
+
"jsonl",
|
|
7
|
+
"ndjson",
|
|
8
|
+
"json-lines",
|
|
9
|
+
"audit",
|
|
10
|
+
"validate",
|
|
11
|
+
"lint",
|
|
12
|
+
"streaming",
|
|
13
|
+
"cli",
|
|
14
|
+
"data-quality",
|
|
15
|
+
"diagnostics",
|
|
16
|
+
"json",
|
|
17
|
+
"file-validation"
|
|
18
|
+
],
|
|
19
|
+
"license": "MIT",
|
|
20
|
+
"type": "module",
|
|
21
|
+
"main": "./src/index.js",
|
|
22
|
+
"exports": {
|
|
23
|
+
".": {
|
|
24
|
+
"import": "./src/index.js",
|
|
25
|
+
"types": "./types/index.d.ts"
|
|
26
|
+
}
|
|
27
|
+
},
|
|
28
|
+
"bin": {
|
|
29
|
+
"jsonl-audit": "bin/jsonl-audit.js"
|
|
30
|
+
},
|
|
31
|
+
"types": "./types/index.d.ts",
|
|
32
|
+
"engines": {
|
|
33
|
+
"node": ">=20.0.0"
|
|
34
|
+
},
|
|
35
|
+
"scripts": {
|
|
36
|
+
"test": "node --test test/validator.test.js test/cli.test.js test/edge-cases.test.js",
|
|
37
|
+
"test:watch": "node --test --watch test/validator.test.js test/cli.test.js test/edge-cases.test.js",
|
|
38
|
+
"check": "node --check src/index.js src/validator.js src/diagnostics.js bin/jsonl-audit.js",
|
|
39
|
+
"pack:dry": "npm pack --dry-run"
|
|
40
|
+
},
|
|
41
|
+
"files": [
|
|
42
|
+
"src/",
|
|
43
|
+
"bin/",
|
|
44
|
+
"types/",
|
|
45
|
+
"docs/",
|
|
46
|
+
"examples/",
|
|
47
|
+
"README.md",
|
|
48
|
+
"CHANGELOG.md",
|
|
49
|
+
"CONTRIBUTING.md",
|
|
50
|
+
"LICENSE"
|
|
51
|
+
],
|
|
52
|
+
"author": "bored-brain",
|
|
53
|
+
"repository": {
|
|
54
|
+
"type": "git",
|
|
55
|
+
"url": "git+https://github.com/bored-brain/jsonl-audit-kit.git"
|
|
56
|
+
},
|
|
57
|
+
"bugs": {
|
|
58
|
+
"url": "https://github.com/bored-brain/jsonl-audit-kit/issues"
|
|
59
|
+
},
|
|
60
|
+
"homepage": "https://github.com/bored-brain/jsonl-audit-kit#readme",
|
|
61
|
+
"devDependencies": {}
|
|
62
|
+
}
|
|
@@ -0,0 +1,124 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* diagnostics.js
|
|
3
|
+
*
|
|
4
|
+
* Defines diagnostic types and helpers for building structured error records.
|
|
5
|
+
* Each diagnostic describes one problem found during JSONL auditing.
|
|
6
|
+
*/
|
|
7
|
+
|
|
8
|
+
/**
|
|
9
|
+
* Enumeration of all diagnostic types produced by jsonl-audit-kit.
|
|
10
|
+
*
|
|
11
|
+
* @enum {string}
|
|
12
|
+
*/
|
|
13
|
+
export const DiagnosticType = {
|
|
14
|
+
/** The line could not be parsed as valid JSON. */
|
|
15
|
+
INVALID_JSON: 'INVALID_JSON',
|
|
16
|
+
|
|
17
|
+
/** The line is entirely blank (whitespace-only). Only emitted when reportBlankLines is true. */
|
|
18
|
+
BLANK_LINE: 'BLANK_LINE',
|
|
19
|
+
|
|
20
|
+
/** The parsed JSON value is not a plain object (e.g. it is an array, string, or number). */
|
|
21
|
+
NOT_AN_OBJECT: 'NOT_AN_OBJECT',
|
|
22
|
+
|
|
23
|
+
/** The parsed JSON value is not an array. Only used when requireArray is true. */
|
|
24
|
+
NOT_AN_ARRAY: 'NOT_AN_ARRAY',
|
|
25
|
+
|
|
26
|
+
/** A required key is missing from the top-level object. */
|
|
27
|
+
MISSING_KEY: 'MISSING_KEY',
|
|
28
|
+
|
|
29
|
+
/** The line is too long (exceeds maxLineBytes). */
|
|
30
|
+
LINE_TOO_LONG: 'LINE_TOO_LONG',
|
|
31
|
+
};
|
|
32
|
+
|
|
33
|
+
/**
|
|
34
|
+
* Build a diagnostic object for an INVALID_JSON error.
|
|
35
|
+
*
|
|
36
|
+
* @param {number} line - 1-based line number.
|
|
37
|
+
* @param {string} message - The JSON parse error message.
|
|
38
|
+
* @returns {{ line: number, type: string, message: string }}
|
|
39
|
+
*/
|
|
40
|
+
export function invalidJson(line, message) {
|
|
41
|
+
return {
|
|
42
|
+
line,
|
|
43
|
+
type: DiagnosticType.INVALID_JSON,
|
|
44
|
+
message: `JSON parse error: ${message}`,
|
|
45
|
+
};
|
|
46
|
+
}
|
|
47
|
+
|
|
48
|
+
/**
|
|
49
|
+
* Build a diagnostic object for a BLANK_LINE.
|
|
50
|
+
*
|
|
51
|
+
* @param {number} line - 1-based line number.
|
|
52
|
+
* @returns {{ line: number, type: string, message: string }}
|
|
53
|
+
*/
|
|
54
|
+
export function blankLine(line) {
|
|
55
|
+
return {
|
|
56
|
+
line,
|
|
57
|
+
type: DiagnosticType.BLANK_LINE,
|
|
58
|
+
message: 'Line is blank',
|
|
59
|
+
};
|
|
60
|
+
}
|
|
61
|
+
|
|
62
|
+
/**
|
|
63
|
+
* Build a diagnostic object for a NOT_AN_OBJECT error.
|
|
64
|
+
*
|
|
65
|
+
* @param {number} line - 1-based line number.
|
|
66
|
+
* @param {string} actualType - The JavaScript type of the parsed value.
|
|
67
|
+
* @returns {{ line: number, type: string, message: string }}
|
|
68
|
+
*/
|
|
69
|
+
export function notAnObject(line, actualType) {
|
|
70
|
+
return {
|
|
71
|
+
line,
|
|
72
|
+
type: DiagnosticType.NOT_AN_OBJECT,
|
|
73
|
+
message: `Expected a JSON object but got ${actualType}`,
|
|
74
|
+
};
|
|
75
|
+
}
|
|
76
|
+
|
|
77
|
+
/**
|
|
78
|
+
* Build a diagnostic object for a NOT_AN_ARRAY error.
|
|
79
|
+
*
|
|
80
|
+
* @param {number} line - 1-based line number.
|
|
81
|
+
* @param {string} actualType - The JavaScript type of the parsed value.
|
|
82
|
+
* @returns {{ line: number, type: string, message: string }}
|
|
83
|
+
*/
|
|
84
|
+
export function notAnArray(line, actualType) {
|
|
85
|
+
return {
|
|
86
|
+
line,
|
|
87
|
+
type: DiagnosticType.NOT_AN_ARRAY,
|
|
88
|
+
message: `Expected a JSON array but got ${actualType}`,
|
|
89
|
+
};
|
|
90
|
+
}
|
|
91
|
+
|
|
92
|
+
/**
|
|
93
|
+
* Build a diagnostic object for a MISSING_KEY error.
|
|
94
|
+
*
|
|
95
|
+
* @param {number} line - 1-based line number.
|
|
96
|
+
* @param {string} key - The key that is absent from the record.
|
|
97
|
+
* @returns {{ line: number, type: string, message: string, key: string }}
|
|
98
|
+
*/
|
|
99
|
+
export function missingKey(line, key) {
|
|
100
|
+
return {
|
|
101
|
+
line,
|
|
102
|
+
type: DiagnosticType.MISSING_KEY,
|
|
103
|
+
message: `Required key "${key}" is missing`,
|
|
104
|
+
key,
|
|
105
|
+
};
|
|
106
|
+
}
|
|
107
|
+
|
|
108
|
+
/**
|
|
109
|
+
* Build a diagnostic object for a LINE_TOO_LONG error.
|
|
110
|
+
*
|
|
111
|
+
* @param {number} line - 1-based line number.
|
|
112
|
+
* @param {number} length - Actual byte length of the line.
|
|
113
|
+
* @param {number} max - Maximum allowed byte length.
|
|
114
|
+
* @returns {{ line: number, type: string, message: string, length: number, max: number }}
|
|
115
|
+
*/
|
|
116
|
+
export function lineTooLong(line, length, max) {
|
|
117
|
+
return {
|
|
118
|
+
line,
|
|
119
|
+
type: DiagnosticType.LINE_TOO_LONG,
|
|
120
|
+
message: `Line is ${length} bytes, exceeds limit of ${max} bytes`,
|
|
121
|
+
length,
|
|
122
|
+
max,
|
|
123
|
+
};
|
|
124
|
+
}
|
package/src/index.js
ADDED
|
@@ -0,0 +1,99 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* index.js — Public API for jsonl-audit-kit
|
|
3
|
+
*
|
|
4
|
+
* This module exposes the two main entry points that library consumers use:
|
|
5
|
+
*
|
|
6
|
+
* auditFile(filePath, options?) — audit a file on disk
|
|
7
|
+
* auditStream(stream, options?) — audit any readable Node.js stream
|
|
8
|
+
*
|
|
9
|
+
* Both functions resolve to an AuditResult object.
|
|
10
|
+
*
|
|
11
|
+
* Example:
|
|
12
|
+
*
|
|
13
|
+
* import { auditFile } from 'jsonl-audit-kit';
|
|
14
|
+
*
|
|
15
|
+
* const result = await auditFile('./events.jsonl', {
|
|
16
|
+
* requireObject: true,
|
|
17
|
+
* requiredKeys: ['id', 'event'],
|
|
18
|
+
* });
|
|
19
|
+
*
|
|
20
|
+
* if (!result.valid) {
|
|
21
|
+
* console.error(`Found ${result.errorCount} error(s)`);
|
|
22
|
+
* for (const err of result.errors) {
|
|
23
|
+
* console.error(` Line ${err.line}: [${err.type}] ${err.message}`);
|
|
24
|
+
* }
|
|
25
|
+
* process.exit(1);
|
|
26
|
+
* }
|
|
27
|
+
*/
|
|
28
|
+
|
|
29
|
+
import { createReadStream } from 'node:fs';
|
|
30
|
+
import { stat } from 'node:fs/promises';
|
|
31
|
+
import { validateOptions } from './utils/validate-options.js';
|
|
32
|
+
import { auditStream as _auditStream } from './validator.js';
|
|
33
|
+
|
|
34
|
+
export { DiagnosticType } from './diagnostics.js';
|
|
35
|
+
|
|
36
|
+
/**
|
|
37
|
+
* Audit a JSONL file at the given path.
|
|
38
|
+
*
|
|
39
|
+
* The file is read using a streaming interface so it is never fully loaded
|
|
40
|
+
* into memory. Only the longest single line needs to fit in the readline
|
|
41
|
+
* buffer.
|
|
42
|
+
*
|
|
43
|
+
* @param {string} filePath - Absolute or relative path to the JSONL file.
|
|
44
|
+
* @param {import('../types/index.d.ts').AuditOptions} [options] - Optional configuration.
|
|
45
|
+
* @returns {Promise<import('../types/index.d.ts').AuditResult>}
|
|
46
|
+
*
|
|
47
|
+
* @throws {TypeError} When options are invalid.
|
|
48
|
+
* @throws {Error} When the file cannot be opened (e.g. ENOENT, EISDIR, EACCES).
|
|
49
|
+
*/
|
|
50
|
+
export async function auditFile(filePath, options = {}) {
|
|
51
|
+
if (typeof filePath !== 'string' || filePath.length === 0) {
|
|
52
|
+
throw new TypeError('filePath must be a non-empty string');
|
|
53
|
+
}
|
|
54
|
+
|
|
55
|
+
const opts = validateOptions(options);
|
|
56
|
+
|
|
57
|
+
// Check existence and that it is a regular file before opening a stream.
|
|
58
|
+
// This surfaces a clearer error than the stream open error.
|
|
59
|
+
let fileStat;
|
|
60
|
+
try {
|
|
61
|
+
fileStat = await stat(filePath);
|
|
62
|
+
} catch (err) {
|
|
63
|
+
const error = new Error(`Cannot access file "${filePath}": ${err.message}`);
|
|
64
|
+
error.code = err.code;
|
|
65
|
+
throw error;
|
|
66
|
+
}
|
|
67
|
+
|
|
68
|
+
if (fileStat.isDirectory()) {
|
|
69
|
+
const error = new Error(`"${filePath}" is a directory, not a file`);
|
|
70
|
+
error.code = 'EISDIR';
|
|
71
|
+
throw error;
|
|
72
|
+
}
|
|
73
|
+
|
|
74
|
+
const stream = createReadStream(filePath, { encoding: 'utf8' });
|
|
75
|
+
|
|
76
|
+
return _auditStream(stream, opts);
|
|
77
|
+
}
|
|
78
|
+
|
|
79
|
+
/**
|
|
80
|
+
* Audit a JSONL readable stream.
|
|
81
|
+
*
|
|
82
|
+
* Use this when you already have a stream (e.g. process.stdin, an HTTP
|
|
83
|
+
* response body, or a custom source) rather than a file path.
|
|
84
|
+
*
|
|
85
|
+
* @param {import('node:stream').Readable} stream - A readable stream of UTF-8 JSONL data.
|
|
86
|
+
* @param {import('../types/index.d.ts').AuditOptions} [options] - Optional configuration.
|
|
87
|
+
* @returns {Promise<import('../types/index.d.ts').AuditResult>}
|
|
88
|
+
*
|
|
89
|
+
* @throws {TypeError} When the stream argument is not a readable stream.
|
|
90
|
+
* @throws {TypeError} When options are invalid.
|
|
91
|
+
*/
|
|
92
|
+
export async function auditStream(stream, options = {}) {
|
|
93
|
+
if (!stream || typeof stream.pipe !== 'function') {
|
|
94
|
+
throw new TypeError('stream must be a readable Node.js stream');
|
|
95
|
+
}
|
|
96
|
+
|
|
97
|
+
const opts = validateOptions(options);
|
|
98
|
+
return _auditStream(stream, opts);
|
|
99
|
+
}
|
|
@@ -0,0 +1,68 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* validate-options.js
|
|
3
|
+
*
|
|
4
|
+
* Validates and normalises user-supplied audit options.
|
|
5
|
+
* Throws a TypeError with a human-readable message on bad input.
|
|
6
|
+
*/
|
|
7
|
+
|
|
8
|
+
/**
|
|
9
|
+
* Default option values.
|
|
10
|
+
*/
|
|
11
|
+
export const DEFAULTS = {
|
|
12
|
+
requireObject: false,
|
|
13
|
+
requireArray: false,
|
|
14
|
+
requiredKeys: [],
|
|
15
|
+
reportBlankLines: false,
|
|
16
|
+
maxErrors: 100,
|
|
17
|
+
maxLineBytes: null,
|
|
18
|
+
};
|
|
19
|
+
|
|
20
|
+
/**
|
|
21
|
+
* Validate and return a fully-resolved options object.
|
|
22
|
+
*
|
|
23
|
+
* @param {object} [userOptions] - Options supplied by the caller.
|
|
24
|
+
* @returns {Required<import('../../types/index.js').AuditOptions>}
|
|
25
|
+
* @throws {TypeError}
|
|
26
|
+
*/
|
|
27
|
+
export function validateOptions(userOptions = {}) {
|
|
28
|
+
if (userOptions === null || typeof userOptions !== 'object' || Array.isArray(userOptions)) {
|
|
29
|
+
throw new TypeError('options must be a plain object');
|
|
30
|
+
}
|
|
31
|
+
|
|
32
|
+
const opts = { ...DEFAULTS, ...userOptions };
|
|
33
|
+
|
|
34
|
+
if (typeof opts.requireObject !== 'boolean') {
|
|
35
|
+
throw new TypeError('options.requireObject must be a boolean');
|
|
36
|
+
}
|
|
37
|
+
if (typeof opts.requireArray !== 'boolean') {
|
|
38
|
+
throw new TypeError('options.requireArray must be a boolean');
|
|
39
|
+
}
|
|
40
|
+
if (opts.requireObject && opts.requireArray) {
|
|
41
|
+
throw new TypeError('options.requireObject and options.requireArray cannot both be true');
|
|
42
|
+
}
|
|
43
|
+
|
|
44
|
+
if (!Array.isArray(opts.requiredKeys)) {
|
|
45
|
+
throw new TypeError('options.requiredKeys must be an array of strings');
|
|
46
|
+
}
|
|
47
|
+
for (const key of opts.requiredKeys) {
|
|
48
|
+
if (typeof key !== 'string' || key.length === 0) {
|
|
49
|
+
throw new TypeError('Each entry in options.requiredKeys must be a non-empty string');
|
|
50
|
+
}
|
|
51
|
+
}
|
|
52
|
+
|
|
53
|
+
if (typeof opts.reportBlankLines !== 'boolean') {
|
|
54
|
+
throw new TypeError('options.reportBlankLines must be a boolean');
|
|
55
|
+
}
|
|
56
|
+
|
|
57
|
+
if (!Number.isInteger(opts.maxErrors) || opts.maxErrors < 1) {
|
|
58
|
+
throw new TypeError('options.maxErrors must be a positive integer');
|
|
59
|
+
}
|
|
60
|
+
|
|
61
|
+
if (opts.maxLineBytes !== null) {
|
|
62
|
+
if (!Number.isInteger(opts.maxLineBytes) || opts.maxLineBytes < 1) {
|
|
63
|
+
throw new TypeError('options.maxLineBytes must be a positive integer or null');
|
|
64
|
+
}
|
|
65
|
+
}
|
|
66
|
+
|
|
67
|
+
return opts;
|
|
68
|
+
}
|