@blamejs/core 0.4.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +230 -0
- package/LICENSE +201 -0
- package/LTS-CALENDAR.md +29 -0
- package/MIGRATING.md +7 -0
- package/NOTICE +59 -0
- package/README.md +100 -0
- package/bin/blamejs.js +13 -0
- package/index.js +253 -0
- package/lib/api-key.js +705 -0
- package/lib/api-snapshot.js +335 -0
- package/lib/app-shutdown.js +381 -0
- package/lib/app.js +364 -0
- package/lib/atomic-file.js +525 -0
- package/lib/audit-chain.js +168 -0
- package/lib/audit-sign.js +319 -0
- package/lib/audit-tools.js +682 -0
- package/lib/audit.js +753 -0
- package/lib/auth/jwt.js +280 -0
- package/lib/auth/oauth.js +691 -0
- package/lib/auth/passkey.js +185 -0
- package/lib/auth/password.js +139 -0
- package/lib/auth/totp.js +17 -0
- package/lib/auth-header.js +81 -0
- package/lib/backup/bundle.js +219 -0
- package/lib/backup/crypto.js +174 -0
- package/lib/backup/index.js +490 -0
- package/lib/backup/manifest.js +275 -0
- package/lib/bundler.js +295 -0
- package/lib/cache.js +819 -0
- package/lib/chain-writer.js +234 -0
- package/lib/cli-helpers.js +201 -0
- package/lib/cli.js +1377 -0
- package/lib/cluster-provider-db.js +245 -0
- package/lib/cluster-storage.js +166 -0
- package/lib/cluster.js +691 -0
- package/lib/consent.js +222 -0
- package/lib/constants.js +186 -0
- package/lib/cookies.js +293 -0
- package/lib/credential-hash.js +303 -0
- package/lib/crypto-field.js +159 -0
- package/lib/crypto.js +250 -0
- package/lib/db-query.js +297 -0
- package/lib/db-schema.js +250 -0
- package/lib/db.js +1054 -0
- package/lib/deprecate.js +226 -0
- package/lib/dev.js +324 -0
- package/lib/error-page.js +424 -0
- package/lib/events.js +135 -0
- package/lib/external-db.js +422 -0
- package/lib/forms.js +378 -0
- package/lib/framework-error.js +189 -0
- package/lib/framework-schema.js +604 -0
- package/lib/handlers.js +350 -0
- package/lib/html-balance.js +227 -0
- package/lib/http-client.js +615 -0
- package/lib/i18n.js +780 -0
- package/lib/jobs.js +181 -0
- package/lib/lazy-require.js +48 -0
- package/lib/log-stream-local.js +137 -0
- package/lib/log-stream-webhook.js +170 -0
- package/lib/log-stream.js +211 -0
- package/lib/log.js +355 -0
- package/lib/mail-bounce.js +507 -0
- package/lib/mail.js +701 -0
- package/lib/metrics.js +647 -0
- package/lib/middleware/api-encrypt.js +553 -0
- package/lib/middleware/attach-user.js +156 -0
- package/lib/middleware/body-parser.js +883 -0
- package/lib/middleware/bot-guard.js +148 -0
- package/lib/middleware/compression.js +436 -0
- package/lib/middleware/cors.js +236 -0
- package/lib/middleware/csp-nonce.js +332 -0
- package/lib/middleware/csrf-protect.js +275 -0
- package/lib/middleware/error-handler.js +46 -0
- package/lib/middleware/health.js +358 -0
- package/lib/middleware/index.js +52 -0
- package/lib/middleware/rate-limit.js +319 -0
- package/lib/middleware/request-id.js +53 -0
- package/lib/middleware/require-auth.js +95 -0
- package/lib/middleware/security-headers.js +91 -0
- package/lib/migrations.js +353 -0
- package/lib/mtls-ca.js +333 -0
- package/lib/mtls-engine-default.js +285 -0
- package/lib/nonce-store.js +177 -0
- package/lib/notify.js +643 -0
- package/lib/ntp-check.js +178 -0
- package/lib/object-store/azure-blob.js +467 -0
- package/lib/object-store/gcs.js +469 -0
- package/lib/object-store/http-put.js +153 -0
- package/lib/object-store/index.js +140 -0
- package/lib/object-store/local.js +163 -0
- package/lib/object-store/retry.js +15 -0
- package/lib/object-store/sigv4.js +535 -0
- package/lib/observability.js +114 -0
- package/lib/pagination.js +371 -0
- package/lib/parsers/index.js +64 -0
- package/lib/parsers/safe-csv.js +224 -0
- package/lib/parsers/safe-env.js +614 -0
- package/lib/parsers/safe-toml.js +745 -0
- package/lib/parsers/safe-xml.js +379 -0
- package/lib/parsers/safe-yaml.js +977 -0
- package/lib/permissions.js +430 -0
- package/lib/pqc-agent.js +85 -0
- package/lib/pqc-gate.js +266 -0
- package/lib/protocol-dispatcher.js +144 -0
- package/lib/queue-local.js +327 -0
- package/lib/queue.js +430 -0
- package/lib/redact.js +192 -0
- package/lib/render.js +193 -0
- package/lib/request-helpers.js +178 -0
- package/lib/restore-bundle.js +239 -0
- package/lib/restore-rollback.js +254 -0
- package/lib/restore.js +301 -0
- package/lib/retry.js +329 -0
- package/lib/router.js +437 -0
- package/lib/safe-async.js +520 -0
- package/lib/safe-buffer.js +162 -0
- package/lib/safe-json.js +532 -0
- package/lib/safe-schema.js +1176 -0
- package/lib/safe-sql.js +157 -0
- package/lib/safe-url.js +109 -0
- package/lib/scheduler.js +680 -0
- package/lib/seeders.js +622 -0
- package/lib/session.js +304 -0
- package/lib/slug.js +243 -0
- package/lib/static.js +268 -0
- package/lib/storage.js +470 -0
- package/lib/subject.js +281 -0
- package/lib/template.js +781 -0
- package/lib/testing.js +621 -0
- package/lib/totp.js +285 -0
- package/lib/tracing.js +484 -0
- package/lib/validate-opts.js +56 -0
- package/lib/vault/index.js +299 -0
- package/lib/vault/passphrase-ops.js +311 -0
- package/lib/vault/passphrase-source.js +198 -0
- package/lib/vault/rotate.js +761 -0
- package/lib/vault/wrap.js +289 -0
- package/lib/vendor/MANIFEST.json +84 -0
- package/lib/vendor/argon2/argon2.cjs +466 -0
- package/lib/vendor/argon2/argon2.d.cts +62 -0
- package/lib/vendor/argon2/package.json +1 -0
- package/lib/vendor/argon2/prebuilds/darwin-arm64/argon2.armv8.glibc.node +0 -0
- package/lib/vendor/argon2/prebuilds/darwin-x64/argon2.glibc.node +0 -0
- package/lib/vendor/argon2/prebuilds/freebsd-arm64/argon2.armv8.glibc.node +0 -0
- package/lib/vendor/argon2/prebuilds/freebsd-x64/argon2.glibc.node +0 -0
- package/lib/vendor/argon2/prebuilds/linux-arm/argon2.armv7.glibc.node +0 -0
- package/lib/vendor/argon2/prebuilds/linux-arm/argon2.armv7.musl.node +0 -0
- package/lib/vendor/argon2/prebuilds/linux-arm64/argon2.armv8.glibc.node +0 -0
- package/lib/vendor/argon2/prebuilds/linux-arm64/argon2.armv8.musl.node +0 -0
- package/lib/vendor/argon2/prebuilds/linux-x64/argon2.glibc.node +0 -0
- package/lib/vendor/argon2/prebuilds/linux-x64/argon2.musl.node +0 -0
- package/lib/vendor/argon2/prebuilds/win32-x64/argon2.glibc.node +0 -0
- package/lib/vendor/noble-ciphers.cjs +9 -0
- package/lib/vendor/pki.cjs +181 -0
- package/lib/vendor/simplewebauthn-server.cjs +328 -0
- package/lib/webhook.js +632 -0
- package/lib/websocket-channels.js +413 -0
- package/lib/websocket.js +833 -0
- package/package.json +39 -0
|
@@ -0,0 +1,371 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
/**
|
|
3
|
+
* pagination — cursor + offset helpers.
|
|
4
|
+
*
|
|
5
|
+
* Every CRUD list endpoint reinvents pagination, usually wrong.
|
|
6
|
+
* The two failure modes:
|
|
7
|
+
*
|
|
8
|
+
* - Offset pagination at depth: `LIMIT n OFFSET 50000` makes the DB
|
|
9
|
+
* scan-and-skip 50,000 rows. O(n). With concurrent writes, rows
|
|
10
|
+
* are also missed/duplicated as new inserts shift the offset.
|
|
11
|
+
* - Cursor pagination without a tie-breaker: `WHERE createdAt > ?`
|
|
12
|
+
* skips or duplicates rows when two records share createdAt.
|
|
13
|
+
*
|
|
14
|
+
* This module ships both, done correctly, plus the encode/decode
|
|
15
|
+
* primitives operators reach for when their SQL doesn't fit the
|
|
16
|
+
* Query-builder shape.
|
|
17
|
+
*
|
|
18
|
+
* Public API:
|
|
19
|
+
*
|
|
20
|
+
* var p = b.pagination;
|
|
21
|
+
*
|
|
22
|
+
* // Cursor: O(1) at any depth. Composite (orderBy, _id) ordering
|
|
23
|
+
* // so ties on the orderBy column are broken by _id and rows are
|
|
24
|
+
* // never skipped.
|
|
25
|
+
* var page = await p.cursor(b.db.from("users"), {
|
|
26
|
+
* cursor: req.query.cursor,
|
|
27
|
+
* limit: req.query.limit,
|
|
28
|
+
* max: 100,
|
|
29
|
+
* default: 25,
|
|
30
|
+
* orderBy: "_id", // default; use "createdAt" etc.
|
|
31
|
+
* direction: "asc", // "asc" | "desc"
|
|
32
|
+
* secret: pageSecret, // Buffer or string; HMAC-tag the cursor
|
|
33
|
+
* });
|
|
34
|
+
* // → { items: [...], nextCursor, prevCursor, limit, hasMore }
|
|
35
|
+
*
|
|
36
|
+
* // Offset: page-numbered. Ergonomic for legacy clients.
|
|
37
|
+
* var off = await p.offset(b.db.from("users"), {
|
|
38
|
+
* page: req.query.page,
|
|
39
|
+
* perPage: req.query.perPage,
|
|
40
|
+
* max: 100,
|
|
41
|
+
* default: 25,
|
|
42
|
+
* });
|
|
43
|
+
* // → { items, total, page, perPage, totalPages, hasMore }
|
|
44
|
+
*
|
|
45
|
+
* // Low-level — for raw SQL or custom row sources.
|
|
46
|
+
* var token = p.encodeCursor({ orderByVal: 12345, id: "abc" }, secret);
|
|
47
|
+
* var state = p.decodeCursor(token, secret);
|
|
48
|
+
*
|
|
49
|
+
* Cursor design:
|
|
50
|
+
* - Composite ordering: (orderBy column, _id). _id is the implicit
|
|
51
|
+
* tie-breaker, so two rows with identical orderByVal are still
|
|
52
|
+
* totally ordered. Forward navigation: WHERE
|
|
53
|
+
* (orderByVal > cur.orderByVal) OR
|
|
54
|
+
* (orderByVal = cur.orderByVal AND _id > cur.id)
|
|
55
|
+
* Backward: same with `<`, then reverse the result set.
|
|
56
|
+
* - Cursors are HMAC-tagged. A tampered cursor is detected at decode
|
|
57
|
+
* time and rejected with PaginationError. Operators MUST pass
|
|
58
|
+
* `secret` (Buffer or string) — there's no auto-derivation, since
|
|
59
|
+
* framework-derived secrets would produce surprises across deploys.
|
|
60
|
+
* - Cursor format: `<base64url state>.<base64url tag>`. State is
|
|
61
|
+
* canonical JSON of `{ v, dir, orderBy, orderByVal, id }`. Tag is
|
|
62
|
+
* SHA3-512(secret || stateJson).slice(0, 16).
|
|
63
|
+
* - direction is part of the cursor — operators don't need to round-
|
|
64
|
+
* trip it via query string. The cursor itself encodes whether it's
|
|
65
|
+
* a "next" or "prev" position so navigation stays consistent.
|
|
66
|
+
*
|
|
67
|
+
* Limit semantics:
|
|
68
|
+
* - Operator passes `default` and `max`. The effective limit is
|
|
69
|
+
* min(max, requestedLimit || default). Negative or non-integer
|
|
70
|
+
* limits are coerced to default.
|
|
71
|
+
* - The page query fetches limit+1 to detect hasMore without a
|
|
72
|
+
* second COUNT(*) trip.
|
|
73
|
+
*
|
|
74
|
+
* Offset is the legacy-client tool, not the recommended path. The
|
|
75
|
+
* module's offset() returns a `total` (from COUNT(*)) and computes
|
|
76
|
+
* `totalPages` so legacy clients can render numbered nav.
|
|
77
|
+
*
|
|
78
|
+
* Out of scope (with structural reasons documented):
|
|
79
|
+
* - Multi-column composite orderBy (orderBy: ["a", "b"]). Use raw
|
|
80
|
+
* SQL + encodeCursor / decodeCursor. The Query builder doesn't
|
|
81
|
+
* model multi-column ORDER BY today.
|
|
82
|
+
* - Cursor TTL / expiry. Operators who want time-limited cursors
|
|
83
|
+
* embed a timestamp in their own state and check at decode-time
|
|
84
|
+
* before passing to .cursor(). The framework's HMAC tag carries
|
|
85
|
+
* no notion of time.
|
|
86
|
+
* - Search / filter integration. Operators chain .where() on the
|
|
87
|
+
* Query before handing to .cursor() — pagination composes with
|
|
88
|
+
* whatever filtering the operator's already applied.
|
|
89
|
+
*/
|
|
90
|
+
|
|
91
|
+
var nodeCrypto = require("node:crypto");
|
|
92
|
+
var { defineClass } = require("./framework-error");
|
|
93
|
+
|
|
94
|
+
var PaginationError = defineClass("PaginationError", { alwaysPermanent: true });
|
|
95
|
+
|
|
96
|
+
var CURSOR_VERSION = 1;
|
|
97
|
+
var TAG_BYTES = 16; // 128-bit HMAC tag truncated from SHA3-512
|
|
98
|
+
var DEFAULT_LIMIT = 25;
|
|
99
|
+
var DEFAULT_MAX_LIMIT = 100;
|
|
100
|
+
|
|
101
|
+
// Canonical JSON — sorted keys at every depth. Mirrors safe-schema /
|
|
102
|
+
// audit-tools so verifier and producer hash exactly the same bytes.
|
|
103
|
+
function _canonicalize(value) {
|
|
104
|
+
if (value === null || typeof value !== "object") return JSON.stringify(value);
|
|
105
|
+
if (Array.isArray(value)) return "[" + value.map(_canonicalize).join(",") + "]";
|
|
106
|
+
var keys = Object.keys(value).sort();
|
|
107
|
+
var parts = [];
|
|
108
|
+
for (var i = 0; i < keys.length; i++) {
|
|
109
|
+
parts.push(JSON.stringify(keys[i]) + ":" + _canonicalize(value[keys[i]]));
|
|
110
|
+
}
|
|
111
|
+
return "{" + parts.join(",") + "}";
|
|
112
|
+
}
|
|
113
|
+
|
|
114
|
+
function _toBuf(secret) {
|
|
115
|
+
if (Buffer.isBuffer(secret)) return secret;
|
|
116
|
+
if (typeof secret === "string") return Buffer.from(secret, "utf8");
|
|
117
|
+
throw new PaginationError("pagination/bad-secret",
|
|
118
|
+
"secret must be a Buffer or non-empty string");
|
|
119
|
+
}
|
|
120
|
+
|
|
121
|
+
function _b64urlEncode(buf) {
|
|
122
|
+
var b = Buffer.isBuffer(buf) ? buf : Buffer.from(buf);
|
|
123
|
+
return b.toString("base64").replace(/\+/g, "-").replace(/\//g, "_").replace(/=+$/, "");
|
|
124
|
+
}
|
|
125
|
+
|
|
126
|
+
function _b64urlDecode(s) {
|
|
127
|
+
if (typeof s !== "string") throw new PaginationError("pagination/bad-cursor", "cursor must be a string");
|
|
128
|
+
var pad = s.length % 4;
|
|
129
|
+
var padded = pad ? s + "=".repeat(4 - pad) : s;
|
|
130
|
+
return Buffer.from(padded.replace(/-/g, "+").replace(/_/g, "/"), "base64");
|
|
131
|
+
}
|
|
132
|
+
|
|
133
|
+
function _tag(secretBuf, stateJson) {
|
|
134
|
+
var h = nodeCrypto.createHash("sha3-512");
|
|
135
|
+
h.update(secretBuf);
|
|
136
|
+
h.update(Buffer.from(stateJson, "utf8"));
|
|
137
|
+
return h.digest().slice(0, TAG_BYTES);
|
|
138
|
+
}
|
|
139
|
+
|
|
140
|
+
function encodeCursor(state, secret) {
|
|
141
|
+
if (!state || typeof state !== "object") {
|
|
142
|
+
throw new PaginationError("pagination/bad-state",
|
|
143
|
+
"encodeCursor: state must be an object");
|
|
144
|
+
}
|
|
145
|
+
var sb = _toBuf(secret);
|
|
146
|
+
if (sb.length === 0) {
|
|
147
|
+
throw new PaginationError("pagination/bad-secret", "secret must be non-empty");
|
|
148
|
+
}
|
|
149
|
+
var withMeta = Object.assign({ v: CURSOR_VERSION }, state);
|
|
150
|
+
var json = _canonicalize(withMeta);
|
|
151
|
+
var tag = _tag(sb, json);
|
|
152
|
+
return _b64urlEncode(json) + "." + _b64urlEncode(tag);
|
|
153
|
+
}
|
|
154
|
+
|
|
155
|
+
function decodeCursor(token, secret) {
|
|
156
|
+
if (typeof token !== "string" || token.length === 0) {
|
|
157
|
+
throw new PaginationError("pagination/bad-cursor", "cursor must be a non-empty string");
|
|
158
|
+
}
|
|
159
|
+
var dot = token.indexOf(".");
|
|
160
|
+
if (dot === -1) {
|
|
161
|
+
throw new PaginationError("pagination/bad-cursor", "cursor missing tag separator");
|
|
162
|
+
}
|
|
163
|
+
var sb = _toBuf(secret);
|
|
164
|
+
var jsonPart = token.slice(0, dot);
|
|
165
|
+
var tagPart = token.slice(dot + 1);
|
|
166
|
+
var json, tag;
|
|
167
|
+
try {
|
|
168
|
+
json = _b64urlDecode(jsonPart).toString("utf8");
|
|
169
|
+
tag = _b64urlDecode(tagPart);
|
|
170
|
+
} catch (_e) {
|
|
171
|
+
throw new PaginationError("pagination/bad-cursor", "cursor base64 decode failed");
|
|
172
|
+
}
|
|
173
|
+
var expected = _tag(sb, json);
|
|
174
|
+
if (tag.length !== expected.length || !nodeCrypto.timingSafeEqual(tag, expected)) {
|
|
175
|
+
throw new PaginationError("pagination/cursor-tag-mismatch",
|
|
176
|
+
"cursor HMAC verification failed (tampered or wrong secret)");
|
|
177
|
+
}
|
|
178
|
+
var state;
|
|
179
|
+
try { state = JSON.parse(json); }
|
|
180
|
+
catch (_e) {
|
|
181
|
+
throw new PaginationError("pagination/bad-cursor", "cursor state JSON malformed");
|
|
182
|
+
}
|
|
183
|
+
if (!state || typeof state !== "object") {
|
|
184
|
+
throw new PaginationError("pagination/bad-cursor", "cursor state is not an object");
|
|
185
|
+
}
|
|
186
|
+
if (state.v !== CURSOR_VERSION) {
|
|
187
|
+
throw new PaginationError("pagination/cursor-version",
|
|
188
|
+
"cursor version " + state.v + " unsupported (current: " + CURSOR_VERSION + ")");
|
|
189
|
+
}
|
|
190
|
+
return state;
|
|
191
|
+
}
|
|
192
|
+
|
|
193
|
+
function _resolveLimit(opts) {
|
|
194
|
+
var max = (typeof opts.max === "number" && opts.max > 0) ? opts.max : DEFAULT_MAX_LIMIT;
|
|
195
|
+
var def = (typeof opts.default === "number" && opts.default > 0) ? opts.default : DEFAULT_LIMIT;
|
|
196
|
+
var requested = parseInt(opts.limit, 10);
|
|
197
|
+
if (isNaN(requested) || requested < 1) requested = def;
|
|
198
|
+
if (requested > max) requested = max;
|
|
199
|
+
return requested;
|
|
200
|
+
}
|
|
201
|
+
|
|
202
|
+
// ---- Cursor pagination ----
|
|
203
|
+
|
|
204
|
+
async function cursor(query, opts) {
|
|
205
|
+
if (!query || typeof query.where !== "function" || typeof query.orderBy !== "function" ||
|
|
206
|
+
typeof query.limit !== "function" || typeof query.all !== "function") {
|
|
207
|
+
throw new PaginationError("pagination/bad-query",
|
|
208
|
+
"cursor: first arg must be a db Query (must support where, orderBy, limit, all)");
|
|
209
|
+
}
|
|
210
|
+
opts = opts || {};
|
|
211
|
+
if (opts.secret == null) {
|
|
212
|
+
throw new PaginationError("pagination/no-secret",
|
|
213
|
+
"cursor: opts.secret is required (Buffer or non-empty string for HMAC tagging)");
|
|
214
|
+
}
|
|
215
|
+
var limit = _resolveLimit(opts);
|
|
216
|
+
var orderBy = typeof opts.orderBy === "string" && opts.orderBy.length > 0 ? opts.orderBy : "_id";
|
|
217
|
+
var direction = (opts.direction === "desc") ? "desc" : "asc";
|
|
218
|
+
|
|
219
|
+
// Decode incoming cursor (if any) and override direction from cursor.
|
|
220
|
+
// The cursor authoritatively encodes which way we're paging — operators
|
|
221
|
+
// shouldn't need to round-trip direction in the URL.
|
|
222
|
+
var cursorState = null;
|
|
223
|
+
var forward = (opts.forward !== false);
|
|
224
|
+
if (opts.cursor) {
|
|
225
|
+
cursorState = decodeCursor(opts.cursor, opts.secret);
|
|
226
|
+
if (cursorState.orderBy !== orderBy || cursorState.dir !== direction) {
|
|
227
|
+
throw new PaginationError("pagination/cursor-mismatch",
|
|
228
|
+
"cursor was created with orderBy='" + cursorState.orderBy + "' direction='" +
|
|
229
|
+
cursorState.dir + "' but call uses orderBy='" + orderBy + "' direction='" +
|
|
230
|
+
direction + "' — operator must use the same opts the cursor was issued under");
|
|
231
|
+
}
|
|
232
|
+
if (typeof cursorState.forward === "boolean") forward = cursorState.forward;
|
|
233
|
+
}
|
|
234
|
+
|
|
235
|
+
// Apply the cursor predicate. Direction interacts with forward/backward:
|
|
236
|
+
// asc + forward → strictly greater than (orderByVal, _id)
|
|
237
|
+
// asc + backward → strictly less than (orderByVal, _id)
|
|
238
|
+
// desc + forward → strictly less than
|
|
239
|
+
// desc + backward → strictly greater than
|
|
240
|
+
// We always SELECT in the direction that matches forward (so the
|
|
241
|
+
// result rows arrive in the right reading order), then if
|
|
242
|
+
// backward we reverse client-side at the end.
|
|
243
|
+
var effectiveAsc = (direction === "asc") === forward; // XNOR
|
|
244
|
+
var compareOp;
|
|
245
|
+
if (cursorState) {
|
|
246
|
+
compareOp = effectiveAsc ? ">" : "<";
|
|
247
|
+
// (orderByVal, _id) <op> (?, ?)
|
|
248
|
+
// Express via OR to be portable across SQLite + Postgres.
|
|
249
|
+
var oCol = orderBy;
|
|
250
|
+
query.whereRaw(
|
|
251
|
+
'"' + oCol + '" ' + compareOp + ' ? OR ("' + oCol + '" = ? AND "_id" ' + compareOp + ' ?)',
|
|
252
|
+
[cursorState.orderByVal, cursorState.orderByVal, cursorState.id]
|
|
253
|
+
);
|
|
254
|
+
}
|
|
255
|
+
query.orderBy(orderBy, effectiveAsc ? "asc" : "desc");
|
|
256
|
+
if (orderBy !== "_id") {
|
|
257
|
+
// Tiebreaker by _id in the same direction — the framework Query
|
|
258
|
+
// only models a single orderBy, so we add the tiebreaker as a
|
|
259
|
+
// raw ORDER BY suffix via _orderLimitOffset cooperation. Today
|
|
260
|
+
// Query lacks multi-orderBy; we emulate by sorting in-memory
|
|
261
|
+
// after the fetch using _id within each orderBy group. Keeps
|
|
262
|
+
// pagination correct without expanding the Query API.
|
|
263
|
+
// No raw orderBy needed because the WHERE condition above
|
|
264
|
+
// strictly disambiguates (orderByVal, _id) tuples — successive
|
|
265
|
+
// pages can't repeat or skip a row even with ties on orderBy.
|
|
266
|
+
}
|
|
267
|
+
query.limit(limit + 1);
|
|
268
|
+
|
|
269
|
+
var rows = await Promise.resolve(query.all());
|
|
270
|
+
|
|
271
|
+
// Tiebreaker stability: when orderBy != _id, the SQL only sorts by
|
|
272
|
+
// orderBy. Within an orderByVal cluster, sort by _id in JS so the
|
|
273
|
+
// cursor predicate's _id-based tiebreaker stays consistent with the
|
|
274
|
+
// returned ordering.
|
|
275
|
+
if (orderBy !== "_id") {
|
|
276
|
+
rows.sort(function (a, b) {
|
|
277
|
+
var av = a[orderBy], bv = b[orderBy];
|
|
278
|
+
if (av < bv) return effectiveAsc ? -1 : 1;
|
|
279
|
+
if (av > bv) return effectiveAsc ? 1 : -1;
|
|
280
|
+
var ai = String(a._id), bi = String(b._id);
|
|
281
|
+
if (ai < bi) return effectiveAsc ? -1 : 1;
|
|
282
|
+
if (ai > bi) return effectiveAsc ? 1 : -1;
|
|
283
|
+
return 0;
|
|
284
|
+
});
|
|
285
|
+
}
|
|
286
|
+
|
|
287
|
+
var hasMore = rows.length > limit;
|
|
288
|
+
var page = hasMore ? rows.slice(0, limit) : rows.slice();
|
|
289
|
+
if (!forward) page.reverse();
|
|
290
|
+
|
|
291
|
+
var nextCursor = null;
|
|
292
|
+
var prevCursor = null;
|
|
293
|
+
if (hasMore && page.length > 0) {
|
|
294
|
+
var last = page[page.length - 1];
|
|
295
|
+
nextCursor = encodeCursor({
|
|
296
|
+
dir: direction, orderBy: orderBy,
|
|
297
|
+
orderByVal: last[orderBy], id: String(last._id),
|
|
298
|
+
forward: true,
|
|
299
|
+
}, opts.secret);
|
|
300
|
+
}
|
|
301
|
+
// Always emit a prev cursor when we have a starting position (the
|
|
302
|
+
// operator was on a non-first page). Operator UI hides it on the
|
|
303
|
+
// first page.
|
|
304
|
+
if (cursorState && page.length > 0) {
|
|
305
|
+
var first = page[0];
|
|
306
|
+
prevCursor = encodeCursor({
|
|
307
|
+
dir: direction, orderBy: orderBy,
|
|
308
|
+
orderByVal: first[orderBy], id: String(first._id),
|
|
309
|
+
forward: false,
|
|
310
|
+
}, opts.secret);
|
|
311
|
+
}
|
|
312
|
+
|
|
313
|
+
return {
|
|
314
|
+
items: page,
|
|
315
|
+
nextCursor: nextCursor,
|
|
316
|
+
prevCursor: prevCursor,
|
|
317
|
+
limit: limit,
|
|
318
|
+
hasMore: hasMore,
|
|
319
|
+
};
|
|
320
|
+
}
|
|
321
|
+
|
|
322
|
+
// ---- Offset pagination ----
|
|
323
|
+
|
|
324
|
+
async function offset(query, opts) {
|
|
325
|
+
if (!query || typeof query.limit !== "function" || typeof query.offset !== "function" ||
|
|
326
|
+
typeof query.all !== "function" || typeof query.count !== "function") {
|
|
327
|
+
throw new PaginationError("pagination/bad-query",
|
|
328
|
+
"offset: first arg must be a db Query (must support limit, offset, all, count)");
|
|
329
|
+
}
|
|
330
|
+
opts = opts || {};
|
|
331
|
+
var perPage = _resolveLimit({ limit: opts.perPage, max: opts.max, default: opts.default });
|
|
332
|
+
var page = parseInt(opts.page, 10);
|
|
333
|
+
if (isNaN(page) || page < 1) page = 1;
|
|
334
|
+
var orderBy = typeof opts.orderBy === "string" && opts.orderBy.length > 0 ? opts.orderBy : "_id";
|
|
335
|
+
var direction = (opts.direction === "desc") ? "desc" : "asc";
|
|
336
|
+
|
|
337
|
+
// Count gives total — required for totalPages calculation. Cheap on
|
|
338
|
+
// an indexed column (which most app tables have via _id).
|
|
339
|
+
var total = await Promise.resolve(query.count());
|
|
340
|
+
|
|
341
|
+
// Build the page query — the operator's existing where() chain is
|
|
342
|
+
// already applied; we just add ordering + limit + offset.
|
|
343
|
+
query.orderBy(orderBy, direction);
|
|
344
|
+
query.limit(perPage);
|
|
345
|
+
query.offset((page - 1) * perPage);
|
|
346
|
+
var items = await Promise.resolve(query.all());
|
|
347
|
+
|
|
348
|
+
var totalPages = total === 0 ? 0 : Math.ceil(total / perPage);
|
|
349
|
+
var hasMore = page < totalPages;
|
|
350
|
+
return {
|
|
351
|
+
items: items,
|
|
352
|
+
total: total,
|
|
353
|
+
page: page,
|
|
354
|
+
perPage: perPage,
|
|
355
|
+
totalPages: totalPages,
|
|
356
|
+
hasMore: hasMore,
|
|
357
|
+
};
|
|
358
|
+
}
|
|
359
|
+
|
|
360
|
+
module.exports = {
|
|
361
|
+
cursor: cursor,
|
|
362
|
+
offset: offset,
|
|
363
|
+
encodeCursor: encodeCursor,
|
|
364
|
+
decodeCursor: decodeCursor,
|
|
365
|
+
PaginationError: PaginationError,
|
|
366
|
+
// Internal helpers exposed for tests
|
|
367
|
+
_resolveLimit: _resolveLimit,
|
|
368
|
+
_b64urlEncode: _b64urlEncode,
|
|
369
|
+
_b64urlDecode: _b64urlDecode,
|
|
370
|
+
CURSOR_VERSION: CURSOR_VERSION,
|
|
371
|
+
};
|
|
@@ -0,0 +1,64 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
/**
|
|
3
|
+
* Multi-format safe parsers — apply the same security defaults blamejs's
|
|
4
|
+
* b.json provides to other common data interchange formats.
|
|
5
|
+
*
|
|
6
|
+
* Currently shipped:
|
|
7
|
+
* xml — RFC-compliant subset; XXE / DOCTYPE / billion-laughs blocked
|
|
8
|
+
* by default; depth + element + attribute count limits;
|
|
9
|
+
* numeric-character-ref bounds checked
|
|
10
|
+
* csv — RFC 4180 parsing + writer with formula-injection prevention
|
|
11
|
+
* (cells starting with =/+/-/@/tab/CR get a single-quote
|
|
12
|
+
* prefix on stringify so Excel doesn't execute them)
|
|
13
|
+
* toml — TOML 1.0 parsing with depth + size limits;
|
|
14
|
+
* prototype-pollution rejection on dotted-key path segments;
|
|
15
|
+
* strict same-key redefinition (silent overwrite would mask
|
|
16
|
+
* config errors operators DO want surfaced); offset
|
|
17
|
+
* date-times decoded as JS Date, local date-time/date/time
|
|
18
|
+
* preserved as ISO strings (no implicit offset assumption);
|
|
19
|
+
* integers > MAX_SAFE_INTEGER rejected so 64-bit values
|
|
20
|
+
* must be encoded as quoted strings
|
|
21
|
+
* env — .env file loader with size cap, schema validation, change
|
|
22
|
+
* tracking via audit chain, and Levenshtein-based typo
|
|
23
|
+
* detection. Rejects $VAR / ${VAR} expansion (consumers
|
|
24
|
+
* reading process.env later wouldn't know whether the
|
|
25
|
+
* value was literal or expanded). Rejects keys outside
|
|
26
|
+
* POSIX shape `^[A-Z_][A-Z0-9_]*$` by default. Schema
|
|
27
|
+
* registration declares per-key sensitivity tiers
|
|
28
|
+
* (boot-only / runtime / breaking) — `breaking` keys
|
|
29
|
+
* refuse silent change unless `{ allow: [...] }` opt-in.
|
|
30
|
+
* Diff result includes `suspicious` field flagging keys
|
|
31
|
+
* that look like typos of registered keys.
|
|
32
|
+
*
|
|
33
|
+
* yaml — YAML 1.2 safe subset (JSON-shaped YAML). Rejects anchors
|
|
34
|
+
* (billion-laughs), aliases (cycles), tags (`!!python/object`-
|
|
35
|
+
* style deserialization), directives, multi-document
|
|
36
|
+
* streams, complex keys, merge keys (`<<`), and tabs in
|
|
37
|
+
* indentation. YAML 1.2 core-schema type inference (NOT
|
|
38
|
+
* 1.1, which had the "Norway problem" where `country: NO`
|
|
39
|
+
* parsed as `country: false`). Block + flow style;
|
|
40
|
+
* literal `|` and folded `>` block scalars with chomp
|
|
41
|
+
* indicators.
|
|
42
|
+
* env — .env file loader with size cap + schema validation;
|
|
43
|
+
* refuses to expand $VAR references; refuses to silently
|
|
44
|
+
* overwrite existing process.env values unless explicitly
|
|
45
|
+
* opted in. Dev-tooling — production secrets should still
|
|
46
|
+
* come through the operator's secrets-management; this is
|
|
47
|
+
* the local-development convenience.
|
|
48
|
+
* ini — Windows .ini files (rare today; lower priority)
|
|
49
|
+
*
|
|
50
|
+
* Public API:
|
|
51
|
+
* parsers.xml.parse(input, opts?) → object
|
|
52
|
+
* parsers.csv.parse(input, opts?) → array
|
|
53
|
+
* parsers.csv.stringify(rows, opts?) → string
|
|
54
|
+
*
|
|
55
|
+
* Error types: each parser exports its own *SafeError class with .code
|
|
56
|
+
* matching the format (xml/..., csv/...).
|
|
57
|
+
*/
|
|
58
|
+
module.exports = {
|
|
59
|
+
xml: require("./safe-xml"),
|
|
60
|
+
csv: require("./safe-csv"),
|
|
61
|
+
toml: require("./safe-toml"),
|
|
62
|
+
yaml: require("./safe-yaml"),
|
|
63
|
+
env: require("./safe-env"),
|
|
64
|
+
};
|
|
@@ -0,0 +1,224 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
/**
|
|
3
|
+
* Security-focused CSV parser + writer.
|
|
4
|
+
*
|
|
5
|
+
* RFC 4180 compliant parsing + the operator-friendly defaults that the
|
|
6
|
+
* RFC doesn't address:
|
|
7
|
+
*
|
|
8
|
+
* - Size + row-count + field-length limits (DoS prevention)
|
|
9
|
+
* - BOM stripping
|
|
10
|
+
* - Configurable delimiter (',' default; '\t' for TSV; ';' for European)
|
|
11
|
+
* - Configurable quote char
|
|
12
|
+
* - CRLF / LF / CR line endings all accepted
|
|
13
|
+
*
|
|
14
|
+
* SECURITY: writer prevents CSV/Excel formula injection. Excel and other
|
|
15
|
+
* spreadsheet apps execute cells starting with '=', '+', '-', '@', or
|
|
16
|
+
* tab/CR characters. By default the writer prefixes such cells with a
|
|
17
|
+
* single quote so the formula doesn't execute when the file is opened.
|
|
18
|
+
* Toggle with { preventFormulaInjection: false } for true RFC 4180 output.
|
|
19
|
+
*
|
|
20
|
+
* Public API:
|
|
21
|
+
* csv.parse(input, opts?) → array of arrays | array of objects
|
|
22
|
+
* csv.stringify(rows, opts?) → string (RFC 4180 + injection-safe)
|
|
23
|
+
* csv.SafeCsvError → error class
|
|
24
|
+
*
|
|
25
|
+
* Defaults (parse):
|
|
26
|
+
* maxBytes: 16 MiB
|
|
27
|
+
* maxRows: 1,000,000
|
|
28
|
+
* maxFieldBytes: 1 MiB
|
|
29
|
+
* delimiter: ','
|
|
30
|
+
* quote: '"'
|
|
31
|
+
* header: true (first row is column names; rows returned as objects)
|
|
32
|
+
* trim: false (don't trim cell whitespace by default)
|
|
33
|
+
*/
|
|
34
|
+
|
|
35
|
+
var C = require("../constants");
|
|
36
|
+
var safeBuffer = require("../safe-buffer");
|
|
37
|
+
var { FrameworkError } = require("../framework-error");
|
|
38
|
+
|
|
39
|
+
class SafeCsvError extends FrameworkError {
|
|
40
|
+
constructor(message, code, position) {
|
|
41
|
+
super(message);
|
|
42
|
+
this.name = "SafeCsvError";
|
|
43
|
+
this.code = code || "csv/invalid";
|
|
44
|
+
this.position = position || null;
|
|
45
|
+
this.isSafeCsvError = true;
|
|
46
|
+
}
|
|
47
|
+
}
|
|
48
|
+
|
|
49
|
+
var DEFAULTS_PARSE = {
|
|
50
|
+
maxBytes: C.BYTES.mib(16),
|
|
51
|
+
maxRows: 1000000,
|
|
52
|
+
maxFieldBytes: C.BYTES.mib(1),
|
|
53
|
+
delimiter: ",",
|
|
54
|
+
quote: '"',
|
|
55
|
+
header: true,
|
|
56
|
+
trim: false,
|
|
57
|
+
};
|
|
58
|
+
|
|
59
|
+
var DEFAULTS_STRINGIFY = {
|
|
60
|
+
delimiter: ",",
|
|
61
|
+
quote: '"',
|
|
62
|
+
preventFormulaInjection: true,
|
|
63
|
+
formulaPrefixChars: ["=", "+", "-", "@", "\t", "\r"],
|
|
64
|
+
always_quote: false, // quote every field; default only quotes when needed
|
|
65
|
+
newline: "\r\n", // RFC 4180
|
|
66
|
+
header: null, // explicit array; null → derive from first object's keys
|
|
67
|
+
};
|
|
68
|
+
|
|
69
|
+
// ---- parse ----
|
|
70
|
+
|
|
71
|
+
function parse(input, opts) {
|
|
72
|
+
opts = Object.assign({}, DEFAULTS_PARSE, opts || {});
|
|
73
|
+
|
|
74
|
+
input = safeBuffer.normalizeText(input, {
|
|
75
|
+
maxBytes: opts.maxBytes,
|
|
76
|
+
errorClass: SafeCsvError,
|
|
77
|
+
typeCode: "csv/wrong-input-type",
|
|
78
|
+
sizeCode: "csv/too-large",
|
|
79
|
+
});
|
|
80
|
+
|
|
81
|
+
var len = input.length;
|
|
82
|
+
var pos = 0;
|
|
83
|
+
var rows = [];
|
|
84
|
+
var row = [];
|
|
85
|
+
var field = "";
|
|
86
|
+
var inQuote = false;
|
|
87
|
+
|
|
88
|
+
function pushField() {
|
|
89
|
+
if (Buffer.byteLength(field, "utf8") > opts.maxFieldBytes) {
|
|
90
|
+
throw new SafeCsvError("field exceeds maxFieldBytes at row " + (rows.length + 1), "csv/field-too-large");
|
|
91
|
+
}
|
|
92
|
+
row.push(opts.trim ? field.trim() : field);
|
|
93
|
+
field = "";
|
|
94
|
+
}
|
|
95
|
+
function pushRow() {
|
|
96
|
+
pushField();
|
|
97
|
+
rows.push(row);
|
|
98
|
+
if (rows.length > opts.maxRows) {
|
|
99
|
+
throw new SafeCsvError("row count exceeds maxRows", "csv/too-many-rows");
|
|
100
|
+
}
|
|
101
|
+
row = [];
|
|
102
|
+
}
|
|
103
|
+
|
|
104
|
+
while (pos < len) {
|
|
105
|
+
var ch = input.charAt(pos);
|
|
106
|
+
if (inQuote) {
|
|
107
|
+
if (ch === opts.quote) {
|
|
108
|
+
// Possible escaped quote (double-quote inside quoted field)
|
|
109
|
+
if (pos + 1 < len && input.charAt(pos + 1) === opts.quote) {
|
|
110
|
+
field += opts.quote;
|
|
111
|
+
pos += 2;
|
|
112
|
+
continue;
|
|
113
|
+
}
|
|
114
|
+
// End of quoted field
|
|
115
|
+
inQuote = false;
|
|
116
|
+
pos += 1;
|
|
117
|
+
continue;
|
|
118
|
+
}
|
|
119
|
+
field += ch;
|
|
120
|
+
pos += 1;
|
|
121
|
+
} else {
|
|
122
|
+
if (ch === opts.delimiter) {
|
|
123
|
+
pushField();
|
|
124
|
+
pos += 1;
|
|
125
|
+
} else if (ch === "\r") {
|
|
126
|
+
// CR or CRLF — both end the row
|
|
127
|
+
pushRow();
|
|
128
|
+
pos += 1;
|
|
129
|
+
if (pos < len && input.charAt(pos) === "\n") pos += 1;
|
|
130
|
+
} else if (ch === "\n") {
|
|
131
|
+
pushRow();
|
|
132
|
+
pos += 1;
|
|
133
|
+
} else if (ch === opts.quote && field === "") {
|
|
134
|
+
inQuote = true;
|
|
135
|
+
pos += 1;
|
|
136
|
+
} else {
|
|
137
|
+
field += ch;
|
|
138
|
+
pos += 1;
|
|
139
|
+
}
|
|
140
|
+
}
|
|
141
|
+
}
|
|
142
|
+
if (inQuote) throw new SafeCsvError("unterminated quoted field", "csv/unterminated-quote");
|
|
143
|
+
// Final row (no trailing newline)
|
|
144
|
+
if (field.length > 0 || row.length > 0) {
|
|
145
|
+
pushRow();
|
|
146
|
+
}
|
|
147
|
+
|
|
148
|
+
if (opts.header) {
|
|
149
|
+
if (rows.length === 0) return [];
|
|
150
|
+
var header = rows[0];
|
|
151
|
+
return rows.slice(1).map(function (r) {
|
|
152
|
+
var obj = {};
|
|
153
|
+
for (var i = 0; i < header.length; i++) obj[header[i]] = r[i] !== undefined ? r[i] : null;
|
|
154
|
+
return obj;
|
|
155
|
+
});
|
|
156
|
+
}
|
|
157
|
+
return rows;
|
|
158
|
+
}
|
|
159
|
+
|
|
160
|
+
// ---- stringify ----
|
|
161
|
+
|
|
162
|
+
function stringify(rows, opts) {
|
|
163
|
+
opts = Object.assign({}, DEFAULTS_STRINGIFY, opts || {});
|
|
164
|
+
if (!Array.isArray(rows)) {
|
|
165
|
+
throw new SafeCsvError("stringify expects an array of rows", "csv/wrong-input-type");
|
|
166
|
+
}
|
|
167
|
+
if (rows.length === 0) return "";
|
|
168
|
+
|
|
169
|
+
// Determine header + row shape
|
|
170
|
+
var header;
|
|
171
|
+
var isObjectRows = false;
|
|
172
|
+
if (Array.isArray(rows[0])) {
|
|
173
|
+
isObjectRows = false;
|
|
174
|
+
header = opts.header || null;
|
|
175
|
+
} else if (typeof rows[0] === "object" && rows[0] !== null) {
|
|
176
|
+
isObjectRows = true;
|
|
177
|
+
header = opts.header || Object.keys(rows[0]);
|
|
178
|
+
} else {
|
|
179
|
+
throw new SafeCsvError("rows must be arrays or objects", "csv/wrong-input-type");
|
|
180
|
+
}
|
|
181
|
+
|
|
182
|
+
function escapeCell(value) {
|
|
183
|
+
var s = value == null ? "" : String(value);
|
|
184
|
+
if (opts.preventFormulaInjection && s.length > 0) {
|
|
185
|
+
var first = s.charAt(0);
|
|
186
|
+
if (opts.formulaPrefixChars.indexOf(first) !== -1) {
|
|
187
|
+
s = "'" + s; // Excel-safe prefix
|
|
188
|
+
}
|
|
189
|
+
}
|
|
190
|
+
var needsQuote = opts.always_quote ||
|
|
191
|
+
s.indexOf(opts.delimiter) !== -1 ||
|
|
192
|
+
s.indexOf(opts.quote) !== -1 ||
|
|
193
|
+
s.indexOf("\n") !== -1 ||
|
|
194
|
+
s.indexOf("\r") !== -1;
|
|
195
|
+
if (needsQuote) {
|
|
196
|
+
s = opts.quote + s.split(opts.quote).join(opts.quote + opts.quote) + opts.quote;
|
|
197
|
+
}
|
|
198
|
+
return s;
|
|
199
|
+
}
|
|
200
|
+
|
|
201
|
+
var out = [];
|
|
202
|
+
if (header) {
|
|
203
|
+
out.push(header.map(escapeCell).join(opts.delimiter));
|
|
204
|
+
}
|
|
205
|
+
for (var i = 0; i < rows.length; i++) {
|
|
206
|
+
var r = rows[i];
|
|
207
|
+
var cells;
|
|
208
|
+
if (isObjectRows) {
|
|
209
|
+
cells = header.map(function (k) { return escapeCell(r[k]); });
|
|
210
|
+
} else {
|
|
211
|
+
cells = r.map(escapeCell);
|
|
212
|
+
}
|
|
213
|
+
out.push(cells.join(opts.delimiter));
|
|
214
|
+
}
|
|
215
|
+
return out.join(opts.newline);
|
|
216
|
+
}
|
|
217
|
+
|
|
218
|
+
module.exports = {
|
|
219
|
+
parse: parse,
|
|
220
|
+
stringify: stringify,
|
|
221
|
+
SafeCsvError: SafeCsvError,
|
|
222
|
+
DEFAULTS_PARSE: DEFAULTS_PARSE,
|
|
223
|
+
DEFAULTS_STRINGIFY: DEFAULTS_STRINGIFY,
|
|
224
|
+
};
|