@absolutejs/audit-s3 0.0.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +162 -0
- package/dist/index.d.ts +113 -0
- package/dist/index.d.ts.map +1 -0
- package/dist/index.js +106 -0
- package/dist/index.js.map +10 -0
- package/package.json +58 -0
package/README.md
ADDED
|
@@ -0,0 +1,162 @@
|
|
|
1
|
+
# @absolutejs/audit-s3
|
|
2
|
+
|
|
3
|
+
S3-compatible `AuditSink` for [@absolutejs/audit](https://github.com/absolutejs/audit).
|
|
4
|
+
|
|
5
|
+
Buffered JSONL writes to AWS S3 / Cloudflare R2 / Backblaze B2 / MinIO — any
|
|
6
|
+
store with a "put a string at a key" API.
|
|
7
|
+
|
|
8
|
+
## Why S3 for audit logs
|
|
9
|
+
|
|
10
|
+
- **WORM (write-once-read-many) buckets** give legal hold for compliance
|
|
11
|
+
retention (SOC2, HIPAA, FedRAMP). The hash-chain in
|
|
12
|
+
`@absolutejs/audit`'s `withIntegrity()` gives tamper-evidence; WORM
|
|
13
|
+
prevents deletion even by an admin.
|
|
14
|
+
- **Lifecycle policies** handle retention windows without a cron job.
|
|
15
|
+
"Move to Glacier after 90 days, delete after 7 years" is one bucket
|
|
16
|
+
policy.
|
|
17
|
+
- **Cheap.** Cold-tier storage costs cents/GB-month.
|
|
18
|
+
- **Queryable later** via Athena, DuckDB, or `s3 ls | xargs cat`.
|
|
19
|
+
|
|
20
|
+
S3 objects are immutable, so the sink buffers events and flushes as JSONL
|
|
21
|
+
files keyed by time. Object keys are lexically sortable; `s3 ls audit/`
|
|
22
|
+
returns events in chronological order.
|
|
23
|
+
|
|
24
|
+
## Install
|
|
25
|
+
|
|
26
|
+
```sh
|
|
27
|
+
bun add @absolutejs/audit @absolutejs/audit-s3
|
|
28
|
+
# Bring whichever S3 client you already use — no SDK lock-in:
|
|
29
|
+
bun add @aws-sdk/client-s3 # OR
|
|
30
|
+
# (Cloudflare R2 Workers binding — no install)
|
|
31
|
+
```
|
|
32
|
+
|
|
33
|
+
## Usage
|
|
34
|
+
|
|
35
|
+
### AWS SDK v3
|
|
36
|
+
|
|
37
|
+
```ts
|
|
38
|
+
import { S3Client, PutObjectCommand } from '@aws-sdk/client-s3';
|
|
39
|
+
import { createAudit, withIntegrity, memorySink } from '@absolutejs/audit';
|
|
40
|
+
import { createS3AuditSink } from '@absolutejs/audit-s3';
|
|
41
|
+
|
|
42
|
+
const s3 = new S3Client({ region: 'us-east-1' });
|
|
43
|
+
|
|
44
|
+
const audit = createAudit({
|
|
45
|
+
sinks: [
|
|
46
|
+
memorySink({ max: 1000 }), // hot tail for queries
|
|
47
|
+
withIntegrity( // tamper-evident
|
|
48
|
+
createS3AuditSink({
|
|
49
|
+
put: async (key, body, contentType) => {
|
|
50
|
+
await s3.send(new PutObjectCommand({
|
|
51
|
+
Bucket: 'my-audit-bucket',
|
|
52
|
+
Key: key,
|
|
53
|
+
Body: body,
|
|
54
|
+
ContentType: contentType,
|
|
55
|
+
}));
|
|
56
|
+
},
|
|
57
|
+
prefix: 'audit/prod/',
|
|
58
|
+
flushIntervalMs: 5_000,
|
|
59
|
+
}),
|
|
60
|
+
{ secret: process.env.AUDIT_SECRET, writerId: 'shard-A' }
|
|
61
|
+
),
|
|
62
|
+
],
|
|
63
|
+
});
|
|
64
|
+
|
|
65
|
+
await audit.append({
|
|
66
|
+
kind: 'auth.login',
|
|
67
|
+
actor: 'user-123',
|
|
68
|
+
metadata: { ip: '10.0.0.1' },
|
|
69
|
+
});
|
|
70
|
+
|
|
71
|
+
// On graceful shutdown:
|
|
72
|
+
await audit.close();
|
|
73
|
+
```
|
|
74
|
+
|
|
75
|
+
### Cloudflare R2 (Workers)
|
|
76
|
+
|
|
77
|
+
```ts
|
|
78
|
+
import { createS3AuditSink } from '@absolutejs/audit-s3';
|
|
79
|
+
|
|
80
|
+
const sink = createS3AuditSink({
|
|
81
|
+
put: async (key, body, contentType) => {
|
|
82
|
+
await env.AUDIT_BUCKET.put(key, body, { httpMetadata: { contentType } });
|
|
83
|
+
},
|
|
84
|
+
});
|
|
85
|
+
```
|
|
86
|
+
|
|
87
|
+
### MinIO
|
|
88
|
+
|
|
89
|
+
Same as AWS SDK — MinIO speaks S3 protocol. Point the `S3Client` at your
|
|
90
|
+
MinIO endpoint and the adapter doesn't care.
|
|
91
|
+
|
|
92
|
+
## Object key layout
|
|
93
|
+
|
|
94
|
+
Default `keyFor` produces:
|
|
95
|
+
|
|
96
|
+
```
|
|
97
|
+
audit/2026-05-30/19-42-15.123-abcd1234.jsonl
|
|
98
|
+
```
|
|
99
|
+
|
|
100
|
+
- **Date prefix** (`2026-05-30/`) — lifecycle policies key off this.
|
|
101
|
+
- **Time component** (`19-42-15.123-`) — UTC `HH-MM-SS.mmm`. Lexical sort
|
|
102
|
+
= chronological order.
|
|
103
|
+
- **8 hex chars random tail** — collision-resistant for two flushes at
|
|
104
|
+
the same millisecond.
|
|
105
|
+
- **`.jsonl`** — one JSON-encoded event per line, trailing newline.
|
|
106
|
+
|
|
107
|
+
Override via the `keyFor` option for tenant-fan-out or hourly partitions.
|
|
108
|
+
|
|
109
|
+
## Flush triggers
|
|
110
|
+
|
|
111
|
+
Whichever fires first:
|
|
112
|
+
|
|
113
|
+
| Trigger | Default | Option |
|
|
114
|
+
|---|---|---|
|
|
115
|
+
| Buffer reaches event count | 1000 | `maxBatchSize` |
|
|
116
|
+
| Buffer reaches byte count | 5_000_000 (5 MB) | `maxBatchBytes` |
|
|
117
|
+
| Time since last flush | 5_000 ms | `flushIntervalMs` |
|
|
118
|
+
| Manual | (caller) | `await sink.flush()` |
|
|
119
|
+
| Close | (caller) | `await sink.close()` |
|
|
120
|
+
|
|
121
|
+
Set `flushIntervalMs: 0` to disable the periodic timer (size-only flushing).
|
|
122
|
+
|
|
123
|
+
## Crash safety
|
|
124
|
+
|
|
125
|
+
Unflushed events are **lost** on process kill. For stricter durability,
|
|
126
|
+
pair the S3 sink with a synchronous sink (Postgres) for critical events
|
|
127
|
+
— S3 is the long-term archive, not the source of truth between flushes.
|
|
128
|
+
Lower `flushIntervalMs` to shrink the loss window at the cost of more
|
|
129
|
+
S3 PUTs.
|
|
130
|
+
|
|
131
|
+
## What this sink does NOT do
|
|
132
|
+
|
|
133
|
+
- **`list` / `prune`** — not implemented. Read audit logs out of S3 via
|
|
134
|
+
Athena / `s3 ls` / DuckDB; enforce retention via S3 lifecycle policies.
|
|
135
|
+
The sink is write-only.
|
|
136
|
+
- **Retry on PUT failure** — `onPutError` callback fires once; the batch
|
|
137
|
+
is dropped. Wire your own retry queue if you need at-least-once.
|
|
138
|
+
- **Multipart upload** — every batch is one PUT. If your batches grow
|
|
139
|
+
past S3's 5GB PutObject limit you have other problems.
|
|
140
|
+
|
|
141
|
+
## Integrity across batches
|
|
142
|
+
|
|
143
|
+
The tamper-evident chain from `withIntegrity()` works across batch
|
|
144
|
+
boundaries automatically. Each event is hashed at append time against
|
|
145
|
+
the prior event's hash; the S3 sink only buffers + flushes — it doesn't
|
|
146
|
+
touch the chain. To verify a chain that spans multiple S3 objects:
|
|
147
|
+
|
|
148
|
+
```ts
|
|
149
|
+
import { verifyChain } from '@absolutejs/audit';
|
|
150
|
+
|
|
151
|
+
// Pull every JSONL object back, sort lexically (= chronologically), flatten:
|
|
152
|
+
const allEvents = orderedJsonlBodies.flatMap(body =>
|
|
153
|
+
body.split('\n').filter(Boolean).map(line => JSON.parse(line))
|
|
154
|
+
);
|
|
155
|
+
const result = await verifyChain(allEvents, secret);
|
|
156
|
+
// { ok: true } or { ok: false, brokenAt: <index> }
|
|
157
|
+
```
|
|
158
|
+
|
|
159
|
+
## License
|
|
160
|
+
|
|
161
|
+
[Apache 2.0](../LICENSE). Substrate-adjacent — rides `@absolutejs/audit`
|
|
162
|
+
(BSL Tier A).
|
package/dist/index.d.ts
ADDED
|
@@ -0,0 +1,113 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* @absolutejs/audit-s3 — S3-compatible `AuditSink` for `@absolutejs/audit`.
|
|
3
|
+
*
|
|
4
|
+
* S3 objects are immutable (no real "append"), so the sink buffers events
|
|
5
|
+
* in memory and flushes them as JSONL files keyed by time. Object keys
|
|
6
|
+
* are lexically sortable so `aws s3 ls` (or any equivalent) returns
|
|
7
|
+
* events in chronological order; the date prefix makes lifecycle policies
|
|
8
|
+
* straightforward ("delete `audit/2026-01-*` after 7 years").
|
|
9
|
+
*
|
|
10
|
+
* **Driver-agnostic.** Takes a narrow `put(key, body)` callback so it
|
|
11
|
+
* works with AWS SDK v3, Cloudflare R2 (via fetch + signed URLs or the
|
|
12
|
+
* R2 Workers binding), Backblaze B2's S3-compatible API, MinIO, or
|
|
13
|
+
* anything else that exposes "put a string at a key" semantics. No
|
|
14
|
+
* `@aws-sdk/client-s3` hard dependency.
|
|
15
|
+
*
|
|
16
|
+
* **Flush triggers** (whichever fires first):
|
|
17
|
+
* - **Size**: `maxBatchSize` events buffered (default 1000) OR
|
|
18
|
+
* `maxBatchBytes` of serialized JSONL (default 5MB).
|
|
19
|
+
* - **Time**: `flushIntervalMs` since last flush (default 5_000).
|
|
20
|
+
* - **Manual**: `await sink.flush()` from the caller.
|
|
21
|
+
* - **Close**: `await sink.close()` drains the buffer + clears the
|
|
22
|
+
* interval timer.
|
|
23
|
+
*
|
|
24
|
+
* **Tamper-evidence works across batches automatically.** The integrity
|
|
25
|
+
* chain (`@absolutejs/audit`'s `withIntegrity()`) computes a hash on
|
|
26
|
+
* every `append()` against the prior event's hash. The S3 sink only
|
|
27
|
+
* buffers + flushes — it doesn't touch the chain. So events split
|
|
28
|
+
* across multiple JSONL files still verify end-to-end via `verifyChain`.
|
|
29
|
+
*
|
|
30
|
+
* **No `list` / `prune`** by design. Reading audit logs out of S3 is a
|
|
31
|
+
* separate operation (Athena, `s3 ls`, etc.); enforcing retention is
|
|
32
|
+
* S3 lifecycle policies. The sink is write-only — pair with a
|
|
33
|
+
* `memorySink` (hot tail for queries) or `audit-postgres` (queryable
|
|
34
|
+
* durable companion) when the host needs `list`.
|
|
35
|
+
*
|
|
36
|
+
* **Crash safety.** Unflushed events are lost on process kill. For
|
|
37
|
+
* stricter durability, pair with a synchronous sink (Postgres) for
|
|
38
|
+
* critical events; the S3 sink is the long-term archive, not the
|
|
39
|
+
* source of truth between flushes. Lower `flushIntervalMs` to shrink
|
|
40
|
+
* the loss window at the cost of more S3 PUTs.
|
|
41
|
+
*/
|
|
42
|
+
import type { AuditEvent, AuditSink } from '@absolutejs/audit';
|
|
43
|
+
/**
|
|
44
|
+
* The single S3 operation the sink needs. Implement against your
|
|
45
|
+
* preferred SDK; the sink calls `put(key, body, contentType)` with
|
|
46
|
+
* `'application/x-ndjson'` and expects a resolved Promise on success
|
|
47
|
+
* or a rejection on failure.
|
|
48
|
+
*/
|
|
49
|
+
export type S3PutFn = (key: string, body: string, contentType: string) => Promise<void>;
|
|
50
|
+
export type S3KeyOptions = {
|
|
51
|
+
/** First event's `at` in the batch. */
|
|
52
|
+
batchStart: number;
|
|
53
|
+
/** Last event's `at` in the batch. */
|
|
54
|
+
batchEnd: number;
|
|
55
|
+
/** How many events the batch contains. */
|
|
56
|
+
eventCount: number;
|
|
57
|
+
};
|
|
58
|
+
export type CreateS3AuditSinkOptions = {
|
|
59
|
+
/**
|
|
60
|
+
* The S3 PUT operation. Receives the object key (relative to whatever
|
|
61
|
+
* bucket your put implementation targets), the serialized JSONL body,
|
|
62
|
+
* and a content-type hint (`'application/x-ndjson'`).
|
|
63
|
+
*/
|
|
64
|
+
put: S3PutFn;
|
|
65
|
+
/**
|
|
66
|
+
* Key prefix. Default `'audit/'`. Useful when one bucket holds
|
|
67
|
+
* multiple audit streams — `'audit/tenant-A/'`, `'audit/system/'`, etc.
|
|
68
|
+
*/
|
|
69
|
+
prefix?: string;
|
|
70
|
+
/**
|
|
71
|
+
* Build the object key from batch metadata. Default produces keys
|
|
72
|
+
* like `audit/2026-05-30/19-42-15.123-abcd1234.jsonl`. The default
|
|
73
|
+
* is intentionally sortable end-to-end so `s3 ls --prefix audit/`
|
|
74
|
+
* returns chronological order; override only when you need a
|
|
75
|
+
* different layout (e.g., per-tenant fan-out, hourly partitions).
|
|
76
|
+
*/
|
|
77
|
+
keyFor?: (params: S3KeyOptions) => string;
|
|
78
|
+
/** Flush when the buffer reaches this many events. Default 1000. */
|
|
79
|
+
maxBatchSize?: number;
|
|
80
|
+
/** Flush when serialized JSONL would exceed this many bytes. Default 5MB. */
|
|
81
|
+
maxBatchBytes?: number;
|
|
82
|
+
/**
|
|
83
|
+
* Flush every N ms regardless of buffer state. Set to `0` or
|
|
84
|
+
* `Infinity` to disable the timer (size/manual-only flushing).
|
|
85
|
+
* Default 5000.
|
|
86
|
+
*/
|
|
87
|
+
flushIntervalMs?: number;
|
|
88
|
+
/**
|
|
89
|
+
* Called when a PUT fails. Default `console.error`. The batch is
|
|
90
|
+
* NOT retried automatically — wire your own retry queue via this
|
|
91
|
+
* hook if you need at-least-once.
|
|
92
|
+
*/
|
|
93
|
+
onPutError?: (error: unknown, key: string, events: AuditEvent[]) => void;
|
|
94
|
+
/** Override `Date.now` for tests. */
|
|
95
|
+
clock?: () => number;
|
|
96
|
+
};
|
|
97
|
+
/**
|
|
98
|
+
* Default object-key generator. Produces lexically-sortable keys of
|
|
99
|
+
* the form `<prefix><YYYY-MM-DD>/<HH-MM-SS>.<ms>-<rand>.jsonl`. The
|
|
100
|
+
* date prefix is the natural lifecycle-policy boundary; the time
|
|
101
|
+
* suffix is unique within the millisecond via a random tail so two
|
|
102
|
+
* flushes at the same instant don't collide.
|
|
103
|
+
*/
|
|
104
|
+
export declare const defaultKeyFor: (prefix: string) => (params: S3KeyOptions) => string;
|
|
105
|
+
declare const DEFAULTS: {
|
|
106
|
+
flushIntervalMs: number;
|
|
107
|
+
maxBatchBytes: number;
|
|
108
|
+
maxBatchSize: number;
|
|
109
|
+
prefix: string;
|
|
110
|
+
};
|
|
111
|
+
export declare const createS3AuditSink: (options: CreateS3AuditSinkOptions) => AuditSink;
|
|
112
|
+
export { DEFAULTS as S3_AUDIT_SINK_DEFAULTS };
|
|
113
|
+
//# sourceMappingURL=index.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAwCG;AACH,OAAO,KAAK,EAAE,UAAU,EAAE,SAAS,EAAE,MAAM,mBAAmB,CAAC;AAE/D;;;;;GAKG;AACH,MAAM,MAAM,OAAO,GAAG,CACrB,GAAG,EAAE,MAAM,EACX,IAAI,EAAE,MAAM,EACZ,WAAW,EAAE,MAAM,KACf,OAAO,CAAC,IAAI,CAAC,CAAC;AAEnB,MAAM,MAAM,YAAY,GAAG;IAC1B,uCAAuC;IACvC,UAAU,EAAE,MAAM,CAAC;IACnB,sCAAsC;IACtC,QAAQ,EAAE,MAAM,CAAC;IACjB,0CAA0C;IAC1C,UAAU,EAAE,MAAM,CAAC;CACnB,CAAC;AAEF,MAAM,MAAM,wBAAwB,GAAG;IACtC;;;;OAIG;IACH,GAAG,EAAE,OAAO,CAAC;IACb;;;OAGG;IACH,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB;;;;;;OAMG;IACH,MAAM,CAAC,EAAE,CAAC,MAAM,EAAE,YAAY,KAAK,MAAM,CAAC;IAC1C,oEAAoE;IACpE,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,6EAA6E;IAC7E,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB;;;;OAIG;IACH,eAAe,CAAC,EAAE,MAAM,CAAC;IACzB;;;;OAIG;IACH,UAAU,CAAC,EAAE,CAAC,KAAK,EAAE,OAAO,EAAE,GAAG,EAAE,MAAM,EAAE,MAAM,EAAE,UAAU,EAAE,KAAK,IAAI,CAAC;IACzE,qCAAqC;IACrC,KAAK,CAAC,EAAE,MAAM,MAAM,CAAC;CACrB,CAAC;AAQF;;;;;;GAMG;AACH,eAAO,MAAM,aAAa,GACxB,QAAQ,MAAM,MACd,QAAQ,YAAY,KAAG,MAUvB,CAAC;AAEH,QAAA,MAAM,QAAQ;;;;;CAKb,CAAC;AAIF,eAAO,MAAM,iBAAiB,GAC7B,SAAS,wBAAwB,KAC/B,SA6GF,CAAC;AAKF,OAAO,EAAE,QAAQ,IAAI,sBAAsB,EAAE,CAAC"}
|
package/dist/index.js
ADDED
|
@@ -0,0 +1,106 @@
|
|
|
1
|
+
// @bun
|
|
2
|
+
// src/index.ts
|
|
3
|
+
var TWO = 2;
|
|
4
|
+
var pad = (n, width) => String(n).padStart(width, "0");
|
|
5
|
+
var defaultKeyFor = (prefix) => (params) => {
|
|
6
|
+
const date = new Date(params.batchStart);
|
|
7
|
+
const datePart = `${date.getUTCFullYear()}-${pad(date.getUTCMonth() + 1, TWO)}-${pad(date.getUTCDate(), TWO)}`;
|
|
8
|
+
const timePart = `${pad(date.getUTCHours(), TWO)}-${pad(date.getUTCMinutes(), TWO)}-${pad(date.getUTCSeconds(), TWO)}.${pad(date.getUTCMilliseconds(), 3)}`;
|
|
9
|
+
const rand = Math.floor(Math.random() * 4294967295).toString(16).padStart(8, "0");
|
|
10
|
+
return `${prefix}${datePart}/${timePart}-${rand}.jsonl`;
|
|
11
|
+
};
|
|
12
|
+
var DEFAULTS = {
|
|
13
|
+
flushIntervalMs: 5000,
|
|
14
|
+
maxBatchBytes: 5000000,
|
|
15
|
+
maxBatchSize: 1000,
|
|
16
|
+
prefix: "audit/"
|
|
17
|
+
};
|
|
18
|
+
var CONTENT_TYPE = "application/x-ndjson";
|
|
19
|
+
var createS3AuditSink = (options) => {
|
|
20
|
+
const prefix = options.prefix ?? DEFAULTS.prefix;
|
|
21
|
+
const maxBatchSize = options.maxBatchSize ?? DEFAULTS.maxBatchSize;
|
|
22
|
+
const maxBatchBytes = options.maxBatchBytes ?? DEFAULTS.maxBatchBytes;
|
|
23
|
+
const flushIntervalMs = options.flushIntervalMs ?? DEFAULTS.flushIntervalMs;
|
|
24
|
+
const keyFor = options.keyFor ?? defaultKeyFor(prefix);
|
|
25
|
+
const clock = options.clock ?? Date.now;
|
|
26
|
+
const onPutError = options.onPutError ?? ((error, key) => console.error(`[audit-s3] PUT failed for "${key}":`, error));
|
|
27
|
+
const { put } = options;
|
|
28
|
+
let buffer = [];
|
|
29
|
+
let bufferedBytes = 0;
|
|
30
|
+
let closed = false;
|
|
31
|
+
let flushChain = Promise.resolve();
|
|
32
|
+
let timer;
|
|
33
|
+
const doFlush = async () => {
|
|
34
|
+
if (buffer.length === 0)
|
|
35
|
+
return;
|
|
36
|
+
const snapshot = buffer;
|
|
37
|
+
buffer = [];
|
|
38
|
+
bufferedBytes = 0;
|
|
39
|
+
const events = snapshot.map((b) => b.event);
|
|
40
|
+
const body = snapshot.map((b) => b.line).join(`
|
|
41
|
+
`) + `
|
|
42
|
+
`;
|
|
43
|
+
const batchStart = events[0].at;
|
|
44
|
+
const batchEnd = events[events.length - 1].at;
|
|
45
|
+
const key = keyFor({
|
|
46
|
+
batchEnd,
|
|
47
|
+
batchStart,
|
|
48
|
+
eventCount: events.length
|
|
49
|
+
});
|
|
50
|
+
try {
|
|
51
|
+
await put(key, body, CONTENT_TYPE);
|
|
52
|
+
} catch (error) {
|
|
53
|
+
onPutError(error, key, events);
|
|
54
|
+
}
|
|
55
|
+
};
|
|
56
|
+
const flush = () => {
|
|
57
|
+
const next = flushChain.then(() => doFlush());
|
|
58
|
+
flushChain = next.catch(() => {});
|
|
59
|
+
return next;
|
|
60
|
+
};
|
|
61
|
+
if (flushIntervalMs > 0 && Number.isFinite(flushIntervalMs) && typeof setInterval !== "undefined") {
|
|
62
|
+
timer = setInterval(() => {
|
|
63
|
+
flush();
|
|
64
|
+
}, flushIntervalMs);
|
|
65
|
+
if (timer && typeof timer.unref === "function") {
|
|
66
|
+
timer.unref();
|
|
67
|
+
}
|
|
68
|
+
}
|
|
69
|
+
return {
|
|
70
|
+
append: (event) => {
|
|
71
|
+
if (closed) {
|
|
72
|
+
throw new Error("[audit-s3] sink is closed");
|
|
73
|
+
}
|
|
74
|
+
const line = JSON.stringify(event);
|
|
75
|
+
const bytes = line.length + 1;
|
|
76
|
+
buffer.push({ bytes, event, line });
|
|
77
|
+
bufferedBytes += bytes;
|
|
78
|
+
if (buffer.length >= maxBatchSize || bufferedBytes >= maxBatchBytes) {
|
|
79
|
+
flush();
|
|
80
|
+
}
|
|
81
|
+
},
|
|
82
|
+
close: async () => {
|
|
83
|
+
if (closed)
|
|
84
|
+
return;
|
|
85
|
+
closed = true;
|
|
86
|
+
if (timer !== undefined) {
|
|
87
|
+
clearInterval(timer);
|
|
88
|
+
timer = undefined;
|
|
89
|
+
}
|
|
90
|
+
await flush();
|
|
91
|
+
await flushChain;
|
|
92
|
+
},
|
|
93
|
+
flush: async () => {
|
|
94
|
+
await flush();
|
|
95
|
+
},
|
|
96
|
+
name: "s3"
|
|
97
|
+
};
|
|
98
|
+
};
|
|
99
|
+
export {
|
|
100
|
+
defaultKeyFor,
|
|
101
|
+
createS3AuditSink,
|
|
102
|
+
DEFAULTS as S3_AUDIT_SINK_DEFAULTS
|
|
103
|
+
};
|
|
104
|
+
|
|
105
|
+
//# debugId=6DDB6A7CA64CA89F64756E2164756E21
|
|
106
|
+
//# sourceMappingURL=index.js.map
|
|
@@ -0,0 +1,10 @@
|
|
|
1
|
+
{
|
|
2
|
+
"version": 3,
|
|
3
|
+
"sources": ["../src/index.ts"],
|
|
4
|
+
"sourcesContent": [
|
|
5
|
+
"/**\n * @absolutejs/audit-s3 — S3-compatible `AuditSink` for `@absolutejs/audit`.\n *\n * S3 objects are immutable (no real \"append\"), so the sink buffers events\n * in memory and flushes them as JSONL files keyed by time. Object keys\n * are lexically sortable so `aws s3 ls` (or any equivalent) returns\n * events in chronological order; the date prefix makes lifecycle policies\n * straightforward (\"delete `audit/2026-01-*` after 7 years\").\n *\n * **Driver-agnostic.** Takes a narrow `put(key, body)` callback so it\n * works with AWS SDK v3, Cloudflare R2 (via fetch + signed URLs or the\n * R2 Workers binding), Backblaze B2's S3-compatible API, MinIO, or\n * anything else that exposes \"put a string at a key\" semantics. No\n * `@aws-sdk/client-s3` hard dependency.\n *\n * **Flush triggers** (whichever fires first):\n * - **Size**: `maxBatchSize` events buffered (default 1000) OR\n * `maxBatchBytes` of serialized JSONL (default 5MB).\n * - **Time**: `flushIntervalMs` since last flush (default 5_000).\n * - **Manual**: `await sink.flush()` from the caller.\n * - **Close**: `await sink.close()` drains the buffer + clears the\n * interval timer.\n *\n * **Tamper-evidence works across batches automatically.** The integrity\n * chain (`@absolutejs/audit`'s `withIntegrity()`) computes a hash on\n * every `append()` against the prior event's hash. The S3 sink only\n * buffers + flushes — it doesn't touch the chain. So events split\n * across multiple JSONL files still verify end-to-end via `verifyChain`.\n *\n * **No `list` / `prune`** by design. Reading audit logs out of S3 is a\n * separate operation (Athena, `s3 ls`, etc.); enforcing retention is\n * S3 lifecycle policies. The sink is write-only — pair with a\n * `memorySink` (hot tail for queries) or `audit-postgres` (queryable\n * durable companion) when the host needs `list`.\n *\n * **Crash safety.** Unflushed events are lost on process kill. For\n * stricter durability, pair with a synchronous sink (Postgres) for\n * critical events; the S3 sink is the long-term archive, not the\n * source of truth between flushes. Lower `flushIntervalMs` to shrink\n * the loss window at the cost of more S3 PUTs.\n */\nimport type { AuditEvent, AuditSink } from '@absolutejs/audit';\n\n/**\n * The single S3 operation the sink needs. Implement against your\n * preferred SDK; the sink calls `put(key, body, contentType)` with\n * `'application/x-ndjson'` and expects a resolved Promise on success\n * or a rejection on failure.\n */\nexport type S3PutFn = (\n\tkey: string,\n\tbody: string,\n\tcontentType: string\n) => Promise<void>;\n\nexport type S3KeyOptions = {\n\t/** First event's `at` in the batch. */\n\tbatchStart: number;\n\t/** Last event's `at` in the batch. */\n\tbatchEnd: number;\n\t/** How many events the batch contains. */\n\teventCount: number;\n};\n\nexport type CreateS3AuditSinkOptions = {\n\t/**\n\t * The S3 PUT operation. Receives the object key (relative to whatever\n\t * bucket your put implementation targets), the serialized JSONL body,\n\t * and a content-type hint (`'application/x-ndjson'`).\n\t */\n\tput: S3PutFn;\n\t/**\n\t * Key prefix. Default `'audit/'`. Useful when one bucket holds\n\t * multiple audit streams — `'audit/tenant-A/'`, `'audit/system/'`, etc.\n\t */\n\tprefix?: string;\n\t/**\n\t * Build the object key from batch metadata. Default produces keys\n\t * like `audit/2026-05-30/19-42-15.123-abcd1234.jsonl`. The default\n\t * is intentionally sortable end-to-end so `s3 ls --prefix audit/`\n\t * returns chronological order; override only when you need a\n\t * different layout (e.g., per-tenant fan-out, hourly partitions).\n\t */\n\tkeyFor?: (params: S3KeyOptions) => string;\n\t/** Flush when the buffer reaches this many events. Default 1000. */\n\tmaxBatchSize?: number;\n\t/** Flush when serialized JSONL would exceed this many bytes. Default 5MB. */\n\tmaxBatchBytes?: number;\n\t/**\n\t * Flush every N ms regardless of buffer state. Set to `0` or\n\t * `Infinity` to disable the timer (size/manual-only flushing).\n\t * Default 5000.\n\t */\n\tflushIntervalMs?: number;\n\t/**\n\t * Called when a PUT fails. Default `console.error`. The batch is\n\t * NOT retried automatically — wire your own retry queue via this\n\t * hook if you need at-least-once.\n\t */\n\tonPutError?: (error: unknown, key: string, events: AuditEvent[]) => void;\n\t/** Override `Date.now` for tests. */\n\tclock?: () => number;\n};\n\nconst TWO = 2;\nconst FOUR = 4;\nconst TEN = 10;\n\nconst pad = (n: number, width: number): string => String(n).padStart(width, '0');\n\n/**\n * Default object-key generator. Produces lexically-sortable keys of\n * the form `<prefix><YYYY-MM-DD>/<HH-MM-SS>.<ms>-<rand>.jsonl`. The\n * date prefix is the natural lifecycle-policy boundary; the time\n * suffix is unique within the millisecond via a random tail so two\n * flushes at the same instant don't collide.\n */\nexport const defaultKeyFor =\n\t(prefix: string) =>\n\t(params: S3KeyOptions): string => {\n\t\tconst date = new Date(params.batchStart);\n\t\tconst datePart = `${date.getUTCFullYear()}-${pad(date.getUTCMonth() + 1, TWO)}-${pad(date.getUTCDate(), TWO)}`;\n\t\tconst timePart = `${pad(date.getUTCHours(), TWO)}-${pad(date.getUTCMinutes(), TWO)}-${pad(date.getUTCSeconds(), TWO)}.${pad(date.getUTCMilliseconds(), 3)}`;\n\t\t// 8 hex chars of randomness — collision probability is\n\t\t// negligible for two flushes at the same millisecond.\n\t\tconst rand = Math.floor(Math.random() * 0xffffffff)\n\t\t\t.toString(16)\n\t\t\t.padStart(8, '0');\n\t\treturn `${prefix}${datePart}/${timePart}-${rand}.jsonl`;\n\t};\n\nconst DEFAULTS = {\n\tflushIntervalMs: 5000,\n\tmaxBatchBytes: 5_000_000,\n\tmaxBatchSize: 1000,\n\tprefix: 'audit/'\n};\n\nconst CONTENT_TYPE = 'application/x-ndjson';\n\nexport const createS3AuditSink = (\n\toptions: CreateS3AuditSinkOptions\n): AuditSink => {\n\tconst prefix = options.prefix ?? DEFAULTS.prefix;\n\tconst maxBatchSize = options.maxBatchSize ?? DEFAULTS.maxBatchSize;\n\tconst maxBatchBytes = options.maxBatchBytes ?? DEFAULTS.maxBatchBytes;\n\tconst flushIntervalMs =\n\t\toptions.flushIntervalMs ?? DEFAULTS.flushIntervalMs;\n\tconst keyFor = options.keyFor ?? defaultKeyFor(prefix);\n\tconst clock = options.clock ?? Date.now;\n\tconst onPutError =\n\t\toptions.onPutError ??\n\t\t((error, key) =>\n\t\t\tconsole.error(`[audit-s3] PUT failed for \"${key}\":`, error));\n\tconst { put } = options;\n\n\ttype Buffered = { event: AuditEvent; line: string; bytes: number };\n\tlet buffer: Buffered[] = [];\n\tlet bufferedBytes = 0;\n\tlet closed = false;\n\t// A single in-flight flush chain so concurrent flush() calls don't\n\t// race on `buffer`. Each flush takes a SNAPSHOT of the buffer and\n\t// clears it synchronously before the PUT awaits.\n\tlet flushChain: Promise<void> = Promise.resolve();\n\tlet timer: ReturnType<typeof setInterval> | undefined;\n\n\tconst doFlush = async (): Promise<void> => {\n\t\tif (buffer.length === 0) return;\n\t\t// Snapshot + clear synchronously. New appends during the PUT\n\t\t// land in a fresh buffer that flushes on the next trigger.\n\t\tconst snapshot = buffer;\n\t\tbuffer = [];\n\t\tbufferedBytes = 0;\n\t\tconst events = snapshot.map((b) => b.event);\n\t\tconst body = snapshot.map((b) => b.line).join('\\n') + '\\n';\n\t\tconst batchStart = events[0]!.at;\n\t\tconst batchEnd = events[events.length - 1]!.at;\n\t\tconst key = keyFor({\n\t\t\tbatchEnd,\n\t\t\tbatchStart,\n\t\t\teventCount: events.length\n\t\t});\n\t\ttry {\n\t\t\tawait put(key, body, CONTENT_TYPE);\n\t\t} catch (error) {\n\t\t\tonPutError(error, key, events);\n\t\t}\n\t};\n\n\t// Chain flushes so a concurrent flush waits for the prior one. A\n\t// throw doesn't poison the chain — `doFlush` already catches via\n\t// `onPutError`.\n\tconst flush = (): Promise<void> => {\n\t\tconst next = flushChain.then(() => doFlush());\n\t\tflushChain = next.catch(() => {});\n\t\treturn next;\n\t};\n\n\t// Initialize the periodic timer if enabled. `unref()` lets the\n\t// process exit even if the timer is pending — flushes during\n\t// shutdown go through `close()` which awaits the final flush.\n\tif (\n\t\tflushIntervalMs > 0 &&\n\t\tNumber.isFinite(flushIntervalMs) &&\n\t\ttypeof setInterval !== 'undefined'\n\t) {\n\t\ttimer = setInterval(() => {\n\t\t\tvoid flush();\n\t\t}, flushIntervalMs);\n\t\t// In Node/Bun, `unref()` exists on the returned Timer object.\n\t\tif (timer && typeof (timer as { unref?: () => void }).unref === 'function') {\n\t\t\t(timer as { unref: () => void }).unref();\n\t\t}\n\t}\n\n\treturn {\n\t\tappend: (event) => {\n\t\t\tif (closed) {\n\t\t\t\tthrow new Error('[audit-s3] sink is closed');\n\t\t\t}\n\t\t\tconst line = JSON.stringify(event);\n\t\t\tconst bytes = line.length + 1; // +1 for the newline\n\t\t\tbuffer.push({ bytes, event, line });\n\t\t\tbufferedBytes += bytes;\n\t\t\t// Synchronous size-based trigger. Fire-and-forget the flush;\n\t\t\t// `flush()` is chained so the next size trigger queues behind\n\t\t\t// the in-flight PUT instead of racing.\n\t\t\tif (\n\t\t\t\tbuffer.length >= maxBatchSize ||\n\t\t\t\tbufferedBytes >= maxBatchBytes\n\t\t\t) {\n\t\t\t\tvoid flush();\n\t\t\t}\n\t\t},\n\t\tclose: async () => {\n\t\t\tif (closed) return;\n\t\t\tclosed = true;\n\t\t\tif (timer !== undefined) {\n\t\t\t\tclearInterval(timer);\n\t\t\t\ttimer = undefined;\n\t\t\t}\n\t\t\t// Final flush — wait for any pending flush PLUS one more so\n\t\t\t// the buffer drains.\n\t\t\tawait flush();\n\t\t\tawait flushChain;\n\t\t},\n\t\tflush: async () => {\n\t\t\tawait flush();\n\t\t},\n\t\tname: 's3'\n\t};\n};\n\n// `clock` is currently only available through the option, not exposed\n// in the returned API. Reserve the export in case a tested-driven user\n// needs it via the default keyFor.\nexport { DEFAULTS as S3_AUDIT_SINK_DEFAULTS };\n"
|
|
6
|
+
],
|
|
7
|
+
"mappings": ";;AAwGA,IAAM,MAAM;AAIZ,IAAM,MAAM,CAAC,GAAW,UAA0B,OAAO,CAAC,EAAE,SAAS,OAAO,GAAG;AASxE,IAAM,gBACZ,CAAC,WACD,CAAC,WAAiC;AAAA,EACjC,MAAM,OAAO,IAAI,KAAK,OAAO,UAAU;AAAA,EACvC,MAAM,WAAW,GAAG,KAAK,eAAe,KAAK,IAAI,KAAK,YAAY,IAAI,GAAG,GAAG,KAAK,IAAI,KAAK,WAAW,GAAG,GAAG;AAAA,EAC3G,MAAM,WAAW,GAAG,IAAI,KAAK,YAAY,GAAG,GAAG,KAAK,IAAI,KAAK,cAAc,GAAG,GAAG,KAAK,IAAI,KAAK,cAAc,GAAG,GAAG,KAAK,IAAI,KAAK,mBAAmB,GAAG,CAAC;AAAA,EAGxJ,MAAM,OAAO,KAAK,MAAM,KAAK,OAAO,IAAI,UAAU,EAChD,SAAS,EAAE,EACX,SAAS,GAAG,GAAG;AAAA,EACjB,OAAO,GAAG,SAAS,YAAY,YAAY;AAAA;AAG7C,IAAM,WAAW;AAAA,EAChB,iBAAiB;AAAA,EACjB,eAAe;AAAA,EACf,cAAc;AAAA,EACd,QAAQ;AACT;AAEA,IAAM,eAAe;AAEd,IAAM,oBAAoB,CAChC,YACe;AAAA,EACf,MAAM,SAAS,QAAQ,UAAU,SAAS;AAAA,EAC1C,MAAM,eAAe,QAAQ,gBAAgB,SAAS;AAAA,EACtD,MAAM,gBAAgB,QAAQ,iBAAiB,SAAS;AAAA,EACxD,MAAM,kBACL,QAAQ,mBAAmB,SAAS;AAAA,EACrC,MAAM,SAAS,QAAQ,UAAU,cAAc,MAAM;AAAA,EACrD,MAAM,QAAQ,QAAQ,SAAS,KAAK;AAAA,EACpC,MAAM,aACL,QAAQ,eACP,CAAC,OAAO,QACR,QAAQ,MAAM,8BAA8B,SAAS,KAAK;AAAA,EAC5D,QAAQ,QAAQ;AAAA,EAGhB,IAAI,SAAqB,CAAC;AAAA,EAC1B,IAAI,gBAAgB;AAAA,EACpB,IAAI,SAAS;AAAA,EAIb,IAAI,aAA4B,QAAQ,QAAQ;AAAA,EAChD,IAAI;AAAA,EAEJ,MAAM,UAAU,YAA2B;AAAA,IAC1C,IAAI,OAAO,WAAW;AAAA,MAAG;AAAA,IAGzB,MAAM,WAAW;AAAA,IACjB,SAAS,CAAC;AAAA,IACV,gBAAgB;AAAA,IAChB,MAAM,SAAS,SAAS,IAAI,CAAC,MAAM,EAAE,KAAK;AAAA,IAC1C,MAAM,OAAO,SAAS,IAAI,CAAC,MAAM,EAAE,IAAI,EAAE,KAAK;AAAA,CAAI,IAAI;AAAA;AAAA,IACtD,MAAM,aAAa,OAAO,GAAI;AAAA,IAC9B,MAAM,WAAW,OAAO,OAAO,SAAS,GAAI;AAAA,IAC5C,MAAM,MAAM,OAAO;AAAA,MAClB;AAAA,MACA;AAAA,MACA,YAAY,OAAO;AAAA,IACpB,CAAC;AAAA,IACD,IAAI;AAAA,MACH,MAAM,IAAI,KAAK,MAAM,YAAY;AAAA,MAChC,OAAO,OAAO;AAAA,MACf,WAAW,OAAO,KAAK,MAAM;AAAA;AAAA;AAAA,EAO/B,MAAM,QAAQ,MAAqB;AAAA,IAClC,MAAM,OAAO,WAAW,KAAK,MAAM,QAAQ,CAAC;AAAA,IAC5C,aAAa,KAAK,MAAM,MAAM,EAAE;AAAA,IAChC,OAAO;AAAA;AAAA,EAMR,IACC,kBAAkB,KAClB,OAAO,SAAS,eAAe,KAC/B,OAAO,gBAAgB,aACtB;AAAA,IACD,QAAQ,YAAY,MAAM;AAAA,MACpB,MAAM;AAAA,OACT,eAAe;AAAA,IAElB,IAAI,SAAS,OAAQ,MAAiC,UAAU,YAAY;AAAA,MAC1E,MAAgC,MAAM;AAAA,IACxC;AAAA,EACD;AAAA,EAEA,OAAO;AAAA,IACN,QAAQ,CAAC,UAAU;AAAA,MAClB,IAAI,QAAQ;AAAA,QACX,MAAM,IAAI,MAAM,2BAA2B;AAAA,MAC5C;AAAA,MACA,MAAM,OAAO,KAAK,UAAU,KAAK;AAAA,MACjC,MAAM,QAAQ,KAAK,SAAS;AAAA,MAC5B,OAAO,KAAK,EAAE,OAAO,OAAO,KAAK,CAAC;AAAA,MAClC,iBAAiB;AAAA,MAIjB,IACC,OAAO,UAAU,gBACjB,iBAAiB,eAChB;AAAA,QACI,MAAM;AAAA,MACZ;AAAA;AAAA,IAED,OAAO,YAAY;AAAA,MAClB,IAAI;AAAA,QAAQ;AAAA,MACZ,SAAS;AAAA,MACT,IAAI,UAAU,WAAW;AAAA,QACxB,cAAc,KAAK;AAAA,QACnB,QAAQ;AAAA,MACT;AAAA,MAGA,MAAM,MAAM;AAAA,MACZ,MAAM;AAAA;AAAA,IAEP,OAAO,YAAY;AAAA,MAClB,MAAM,MAAM;AAAA;AAAA,IAEb,MAAM;AAAA,EACP;AAAA;",
|
|
8
|
+
"debugId": "6DDB6A7CA64CA89F64756E2164756E21",
|
|
9
|
+
"names": []
|
|
10
|
+
}
|
package/package.json
ADDED
|
@@ -0,0 +1,58 @@
|
|
|
1
|
+
{
|
|
2
|
+
"name": "@absolutejs/audit-s3",
|
|
3
|
+
"version": "0.0.1",
|
|
4
|
+
"description": "S3-compatible AuditSink for @absolutejs/audit. Buffered JSONL writes to AWS S3 / Cloudflare R2 / Backblaze B2 / MinIO. Time-sortable object keys; WORM-bucket-friendly for compliance retention.",
|
|
5
|
+
"repository": {
|
|
6
|
+
"type": "git",
|
|
7
|
+
"url": "https://github.com/absolutejs/audit-adapters.git",
|
|
8
|
+
"directory": "s3"
|
|
9
|
+
},
|
|
10
|
+
"main": "./dist/index.js",
|
|
11
|
+
"module": "./dist/index.js",
|
|
12
|
+
"types": "./dist/index.d.ts",
|
|
13
|
+
"type": "module",
|
|
14
|
+
"license": "Apache-2.0",
|
|
15
|
+
"author": "Alex Kahn",
|
|
16
|
+
"publishConfig": {
|
|
17
|
+
"access": "public"
|
|
18
|
+
},
|
|
19
|
+
"keywords": [
|
|
20
|
+
"audit",
|
|
21
|
+
"s3",
|
|
22
|
+
"r2",
|
|
23
|
+
"cloudflare",
|
|
24
|
+
"minio",
|
|
25
|
+
"backblaze",
|
|
26
|
+
"absolutejs",
|
|
27
|
+
"audit-log",
|
|
28
|
+
"compliance",
|
|
29
|
+
"worm"
|
|
30
|
+
],
|
|
31
|
+
"scripts": {
|
|
32
|
+
"build": "rm -rf dist && bun build src/index.ts --outdir dist --sourcemap --target=bun --external @absolutejs/audit && tsc --project tsconfig.build.json",
|
|
33
|
+
"test": "bun test",
|
|
34
|
+
"typecheck": "tsc --noEmit",
|
|
35
|
+
"format": "prettier --write \"./**/*.{ts,json,md}\"",
|
|
36
|
+
"release": "bun run format && bun run build && bun publish"
|
|
37
|
+
},
|
|
38
|
+
"peerDependencies": {
|
|
39
|
+
"@absolutejs/audit": ">= 0.0.1"
|
|
40
|
+
},
|
|
41
|
+
"devDependencies": {
|
|
42
|
+
"@absolutejs/audit": "^0.0.1",
|
|
43
|
+
"@types/bun": "1.3.14",
|
|
44
|
+
"prettier": "3.5.3",
|
|
45
|
+
"typescript": "5.8.3"
|
|
46
|
+
},
|
|
47
|
+
"exports": {
|
|
48
|
+
".": {
|
|
49
|
+
"types": "./dist/index.d.ts",
|
|
50
|
+
"import": "./dist/index.js",
|
|
51
|
+
"default": "./dist/index.js"
|
|
52
|
+
}
|
|
53
|
+
},
|
|
54
|
+
"files": [
|
|
55
|
+
"dist",
|
|
56
|
+
"README.md"
|
|
57
|
+
]
|
|
58
|
+
}
|