@zlink-systems/framework-locations-redis 0.26.0 → 0.28.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/.tsbuildinfo +1 -1
- package/dist/framework-internal.d.ts +1 -0
- package/dist/framework-internal.js +12 -0
- package/dist/opaque-redis-scripts.d.ts +17 -2
- package/dist/opaque-redis-scripts.js +126 -114
- package/dist/opaque-store.js +43 -32
- package/dist/redis-connection.d.ts +1 -1
- package/dist/relocation-store.js +8 -7
- package/package.json +3 -3
- package/src/framework-internal.ts +20 -0
- package/src/opaque-redis-scripts.ts +135 -116
- package/src/opaque-store.ts +77 -37
- package/src/redis-connection.ts +1 -1
- package/src/relocation-store.ts +17 -7
|
@@ -0,0 +1 @@
|
|
|
1
|
+
export declare const ZLINK_PROVIDER_MAX_KEY_BYTES: number, ZLINK_PROVIDER_MAX_VALUE_BYTES: number, ZLINK_PROVIDER_MAX_VERSION_BYTES: number, ZLINK_PROVIDER_MAX_WRITE_KEYS: number, ZLINK_PROVIDER_MAX_WRITE_BYTES: number, ZLINK_PROVIDER_MAX_ENCODED_PAGE_BYTES: number, ZLINK_PROVIDER_MAX_SCAN_CURSOR_BYTES: number, ZLINK_PROVIDER_MAX_PAGE_SIZE: number, ZLINK_RELOCATION_MAX_BLOB_REFERENCE_BYTES: number;
|
|
@@ -0,0 +1,12 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
var __importDefault = (this && this.__importDefault) || function (mod) {
|
|
3
|
+
return (mod && mod.__esModule) ? mod : { "default": mod };
|
|
4
|
+
};
|
|
5
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
6
|
+
exports.ZLINK_RELOCATION_MAX_BLOB_REFERENCE_BYTES = exports.ZLINK_PROVIDER_MAX_PAGE_SIZE = exports.ZLINK_PROVIDER_MAX_SCAN_CURSOR_BYTES = exports.ZLINK_PROVIDER_MAX_ENCODED_PAGE_BYTES = exports.ZLINK_PROVIDER_MAX_WRITE_BYTES = exports.ZLINK_PROVIDER_MAX_WRITE_KEYS = exports.ZLINK_PROVIDER_MAX_VERSION_BYTES = exports.ZLINK_PROVIDER_MAX_VALUE_BYTES = exports.ZLINK_PROVIDER_MAX_KEY_BYTES = void 0;
|
|
7
|
+
const node_module_1 = require("node:module");
|
|
8
|
+
const node_path_1 = __importDefault(require("node:path"));
|
|
9
|
+
const requireFramework = (0, node_module_1.createRequire)(__filename);
|
|
10
|
+
const frameworkEntry = requireFramework.resolve('@zlink-systems/framework');
|
|
11
|
+
const internal = requireFramework(node_path_1.default.join(node_path_1.default.dirname(frameworkEntry), 'internal'));
|
|
12
|
+
exports.ZLINK_PROVIDER_MAX_KEY_BYTES = internal.ZLINK_PROVIDER_MAX_KEY_BYTES, exports.ZLINK_PROVIDER_MAX_VALUE_BYTES = internal.ZLINK_PROVIDER_MAX_VALUE_BYTES, exports.ZLINK_PROVIDER_MAX_VERSION_BYTES = internal.ZLINK_PROVIDER_MAX_VERSION_BYTES, exports.ZLINK_PROVIDER_MAX_WRITE_KEYS = internal.ZLINK_PROVIDER_MAX_WRITE_KEYS, exports.ZLINK_PROVIDER_MAX_WRITE_BYTES = internal.ZLINK_PROVIDER_MAX_WRITE_BYTES, exports.ZLINK_PROVIDER_MAX_ENCODED_PAGE_BYTES = internal.ZLINK_PROVIDER_MAX_ENCODED_PAGE_BYTES, exports.ZLINK_PROVIDER_MAX_SCAN_CURSOR_BYTES = internal.ZLINK_PROVIDER_MAX_SCAN_CURSOR_BYTES, exports.ZLINK_PROVIDER_MAX_PAGE_SIZE = internal.ZLINK_PROVIDER_MAX_PAGE_SIZE, exports.ZLINK_RELOCATION_MAX_BLOB_REFERENCE_BYTES = internal.ZLINK_RELOCATION_MAX_BLOB_REFERENCE_BYTES;
|
|
@@ -1,5 +1,20 @@
|
|
|
1
|
-
export declare const
|
|
2
|
-
|
|
1
|
+
export declare const REDIS_STORE_TOKEN: Readonly<{
|
|
2
|
+
readonly Conflict: "conflict";
|
|
3
|
+
readonly Backlog: "backlog";
|
|
4
|
+
readonly Applied: "applied";
|
|
5
|
+
readonly Expired: "expired";
|
|
6
|
+
readonly Capacity: "capacity";
|
|
7
|
+
readonly Page: "page";
|
|
8
|
+
readonly AlreadyStored: "alreadyStored";
|
|
9
|
+
readonly Stored: "stored";
|
|
10
|
+
readonly Missing: "missing";
|
|
11
|
+
readonly Value: "value";
|
|
12
|
+
readonly Version: "version";
|
|
13
|
+
readonly Put: "put";
|
|
14
|
+
readonly Delete: "delete";
|
|
15
|
+
}>;
|
|
16
|
+
export declare const OPAQUE_READ_SCRIPT: Buffer<ArrayBuffer>;
|
|
17
|
+
export declare const OPAQUE_WRITE_SCRIPT: Buffer<ArrayBuffer>;
|
|
3
18
|
export declare const OPAQUE_SCAN_START_SCRIPT: string;
|
|
4
19
|
export declare const OPAQUE_SCAN_CONTINUE_SCRIPT: string;
|
|
5
20
|
export declare const BLOB_PUT_SCRIPT: string;
|
|
@@ -1,10 +1,39 @@
|
|
|
1
1
|
"use strict";
|
|
2
2
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
-
exports.BLOB_RENEW_SCRIPT = exports.BLOB_READ_SCRIPT = exports.BLOB_PUT_SCRIPT = exports.OPAQUE_SCAN_CONTINUE_SCRIPT = exports.OPAQUE_SCAN_START_SCRIPT = exports.OPAQUE_WRITE_SCRIPT = exports.OPAQUE_READ_SCRIPT = void 0;
|
|
3
|
+
exports.BLOB_RENEW_SCRIPT = exports.BLOB_READ_SCRIPT = exports.BLOB_PUT_SCRIPT = exports.OPAQUE_SCAN_CONTINUE_SCRIPT = exports.OPAQUE_SCAN_START_SCRIPT = exports.OPAQUE_WRITE_SCRIPT = exports.OPAQUE_READ_SCRIPT = exports.REDIS_STORE_TOKEN = void 0;
|
|
4
|
+
const framework_internal_1 = require("./framework-internal");
|
|
5
|
+
const REDIS_MILLISECONDS_PER_SECOND = 1000;
|
|
6
|
+
const REDIS_MICROSECONDS_PER_MILLISECOND = 1000;
|
|
7
|
+
const REDIS_CLEANUP_RETRY_DELAY_MS = 1000;
|
|
8
|
+
const REDIS_EXPIRED_SNAPSHOT_BATCH_SIZE = 128;
|
|
9
|
+
const REDIS_ROW_COMPACTION_ENTRY_LIMIT = 128;
|
|
10
|
+
const REDIS_SCAN_MINIMUM_WORK_BUDGET = 128;
|
|
11
|
+
const REDIS_ENCODED_ITEM_OVERHEAD_BYTES = 128;
|
|
12
|
+
const REDIS_SCAN_WORK_BUDGET_MULTIPLIER = 4;
|
|
13
|
+
const REDIS_CLEANUP_BATCH_SIZE = 32;
|
|
14
|
+
const REDIS_RECORD_RETENTION_MS = 60000;
|
|
15
|
+
const REDIS_SCAN_SNAPSHOT_RETENTION_MS = 60000;
|
|
16
|
+
const REDIS_MAX_SCAN_SNAPSHOTS = 4096;
|
|
17
|
+
const REDIS_UTF8_PREFIX_UPPER_BOUND_BYTE = 0xff;
|
|
18
|
+
exports.REDIS_STORE_TOKEN = Object.freeze({
|
|
19
|
+
Conflict: 'conflict',
|
|
20
|
+
Backlog: 'backlog',
|
|
21
|
+
Applied: 'applied',
|
|
22
|
+
Expired: 'expired',
|
|
23
|
+
Capacity: 'capacity',
|
|
24
|
+
Page: 'page',
|
|
25
|
+
AlreadyStored: 'alreadyStored',
|
|
26
|
+
Stored: 'stored',
|
|
27
|
+
Missing: 'missing',
|
|
28
|
+
Value: 'value',
|
|
29
|
+
Version: 'version',
|
|
30
|
+
Put: 'put',
|
|
31
|
+
Delete: 'delete'
|
|
32
|
+
});
|
|
4
33
|
const PROLOGUE = `
|
|
5
34
|
if redis.replicate_commands then redis.replicate_commands() end
|
|
6
35
|
local time = redis.call('TIME')
|
|
7
|
-
local nowMs = tonumber(time[1]) *
|
|
36
|
+
local nowMs = tonumber(time[1]) * ${REDIS_MILLISECONDS_PER_SECOND} + math.floor(tonumber(time[2]) / ${REDIS_MICROSECONDS_PER_MILLISECOND})
|
|
8
37
|
`;
|
|
9
38
|
// Shared decode helper. Every opaque row is a Redis ZSET append-log; the
|
|
10
39
|
// member with the highest score (the provider's monotonically increasing
|
|
@@ -39,34 +68,40 @@ local function encodeMember(originalKey, bytes, version, expiresAtMs, tombstone)
|
|
|
39
68
|
})
|
|
40
69
|
end
|
|
41
70
|
`;
|
|
42
|
-
exports.OPAQUE_READ_SCRIPT = PROLOGUE +
|
|
71
|
+
exports.OPAQUE_READ_SCRIPT = Buffer.from(PROLOGUE +
|
|
43
72
|
DECODE_HELPERS +
|
|
44
73
|
`
|
|
45
74
|
local record = liveRecordAt(KEYS[1], nowMs)
|
|
46
75
|
if not record then return {0, nowMs} end
|
|
47
76
|
return {1, nowMs, record[1], record[2], record[3], tostring(tonumber(record[4]))}
|
|
48
|
-
|
|
77
|
+
`);
|
|
49
78
|
// KEYS[1..6] = indexKey, mapKey, cleanupKey, sequenceKey, snapshotExpiryKey,
|
|
50
79
|
// snapshotBoundaryKey (private auxiliary keys, not part of the public
|
|
51
80
|
// contract). KEYS[7..] = the opaque record row keys referenced by
|
|
52
81
|
// conditions/mutations, in the order fixed by ARGV[1]/ARGV[2].
|
|
53
82
|
// ARGV[1] = JSON conditions: ['missing', keyIndex, originalKey]
|
|
54
83
|
// | ['version', keyIndex, originalKey, expectedVersion]
|
|
84
|
+
// | ['value', keyIndex, originalKey, expectedBytesArgIndex]
|
|
55
85
|
// ARGV[2] = JSON mutation metadata (no raw bytes, JSON must stay text-safe):
|
|
56
86
|
// ['put', keyIndex, originalKey, retentionMsOrFalse]
|
|
57
87
|
// | ['delete', keyIndex, originalKey]
|
|
58
|
-
// ARGV[3..] =
|
|
59
|
-
|
|
60
|
-
|
|
88
|
+
// ARGV[3..] = raw expected bytes for 'value' conditions, then raw bytes for
|
|
89
|
+
// 'put' mutations, each in request order.
|
|
90
|
+
const PRUNE_HISTORY_HELPER = `
|
|
91
|
+
local function pruneHistory(rowKey, minimumBoundary)
|
|
92
|
+
if minimumBoundary then
|
|
93
|
+
local anchor = redis.call('ZREVRANGEBYSCORE', rowKey, minimumBoundary, '-inf', 'WITHSCORES', 'LIMIT', 0, 1)
|
|
94
|
+
if #anchor == 2 then
|
|
95
|
+
redis.call('ZREMRANGEBYSCORE', rowKey, '-inf', '(' .. anchor[2])
|
|
96
|
+
end
|
|
97
|
+
else
|
|
98
|
+
redis.call('ZREMRANGEBYRANK', rowKey, 0, -2)
|
|
99
|
+
end
|
|
100
|
+
end
|
|
101
|
+
`;
|
|
102
|
+
const CLEANUP_AND_BOUNDARY = PRUNE_HISTORY_HELPER +
|
|
61
103
|
`
|
|
62
|
-
local
|
|
63
|
-
local mapKey = KEYS[2]
|
|
64
|
-
local cleanupKey = KEYS[3]
|
|
65
|
-
local sequenceKey = KEYS[4]
|
|
66
|
-
local snapshotExpiryKey = KEYS[5]
|
|
67
|
-
local snapshotBoundaryKey = KEYS[6]
|
|
68
|
-
|
|
69
|
-
local expiredSnapshots = redis.call('ZRANGEBYSCORE', snapshotExpiryKey, '-inf', nowMs, 'LIMIT', 0, 128)
|
|
104
|
+
local expiredSnapshots = redis.call('ZRANGEBYSCORE', snapshotExpiryKey, '-inf', nowMs, 'LIMIT', 0, ${REDIS_EXPIRED_SNAPSHOT_BATCH_SIZE})
|
|
70
105
|
for _, snapshotId in ipairs(expiredSnapshots) do
|
|
71
106
|
redis.call('ZREM', snapshotExpiryKey, snapshotId)
|
|
72
107
|
redis.call('ZREM', snapshotBoundaryKey, snapshotId)
|
|
@@ -75,7 +110,7 @@ local minimumBoundary = nil
|
|
|
75
110
|
local boundaryEntry = redis.call('ZRANGE', snapshotBoundaryKey, 0, 0, 'WITHSCORES')
|
|
76
111
|
if #boundaryEntry == 2 then minimumBoundary = tonumber(boundaryEntry[2]) end
|
|
77
112
|
|
|
78
|
-
local due = redis.call('ZRANGEBYSCORE', cleanupKey, '-inf', nowMs, 'LIMIT', 0,
|
|
113
|
+
local due = redis.call('ZRANGEBYSCORE', cleanupKey, '-inf', nowMs, 'LIMIT', 0, ${REDIS_CLEANUP_BATCH_SIZE})
|
|
79
114
|
for _, original in ipairs(due) do
|
|
80
115
|
local rowKey = redis.call('HGET', mapKey, original)
|
|
81
116
|
local members = {}
|
|
@@ -87,23 +122,20 @@ for _, original in ipairs(due) do
|
|
|
87
122
|
redis.call('HDEL', mapKey, original)
|
|
88
123
|
redis.call('ZREM', cleanupKey, original)
|
|
89
124
|
elseif minimumBoundary then
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
redis.call('ZREMRANGEBYSCORE', rowKey, '-inf', '(' .. anchor[2])
|
|
93
|
-
end
|
|
94
|
-
redis.call('ZADD', cleanupKey, nowMs + 1000, original)
|
|
125
|
+
pruneHistory(rowKey, minimumBoundary)
|
|
126
|
+
redis.call('ZADD', cleanupKey, nowMs + ${REDIS_CLEANUP_RETRY_DELAY_MS}, original)
|
|
95
127
|
else
|
|
96
128
|
local record = decodeMember(members[1])
|
|
97
129
|
local expiresAtMs = tonumber(record[4])
|
|
98
|
-
if record[5] == true or (expiresAtMs > 0 and expiresAtMs +
|
|
130
|
+
if record[5] == true or (expiresAtMs > 0 and expiresAtMs + ${REDIS_RECORD_RETENTION_MS} <= nowMs) then
|
|
99
131
|
redis.call('DEL', rowKey)
|
|
100
132
|
redis.call('ZREM', indexKey, original)
|
|
101
133
|
redis.call('HDEL', mapKey, original)
|
|
102
134
|
redis.call('ZREM', cleanupKey, original)
|
|
103
135
|
else
|
|
104
|
-
|
|
136
|
+
pruneHistory(rowKey, minimumBoundary)
|
|
105
137
|
if expiresAtMs > 0 then
|
|
106
|
-
redis.call('ZADD', cleanupKey, math.max(nowMs +
|
|
138
|
+
redis.call('ZADD', cleanupKey, math.max(nowMs + ${REDIS_CLEANUP_RETRY_DELAY_MS}, expiresAtMs + ${REDIS_RECORD_RETENTION_MS}), original)
|
|
107
139
|
else
|
|
108
140
|
redis.call('ZREM', cleanupKey, original)
|
|
109
141
|
end
|
|
@@ -111,36 +143,55 @@ for _, original in ipairs(due) do
|
|
|
111
143
|
end
|
|
112
144
|
end
|
|
113
145
|
|
|
146
|
+
`;
|
|
147
|
+
exports.OPAQUE_WRITE_SCRIPT = Buffer.from(PROLOGUE +
|
|
148
|
+
DECODE_HELPERS +
|
|
149
|
+
`
|
|
150
|
+
local indexKey = KEYS[1]
|
|
151
|
+
local mapKey = KEYS[2]
|
|
152
|
+
local cleanupKey = KEYS[3]
|
|
153
|
+
local sequenceKey = KEYS[4]
|
|
154
|
+
local snapshotExpiryKey = KEYS[5]
|
|
155
|
+
local snapshotBoundaryKey = KEYS[6]
|
|
156
|
+
|
|
157
|
+
` +
|
|
158
|
+
CLEANUP_AND_BOUNDARY +
|
|
159
|
+
`
|
|
114
160
|
local conditions = cjson.decode(ARGV[1])
|
|
115
161
|
local mutations = cjson.decode(ARGV[2])
|
|
116
162
|
|
|
117
163
|
for _, condition in ipairs(conditions) do
|
|
118
164
|
local record = liveRecordAt(KEYS[condition[2] + 6], nowMs)
|
|
119
165
|
local currentVersion = record and record[3] or nil
|
|
120
|
-
if condition[1] == '
|
|
121
|
-
if currentVersion ~= nil then return {'
|
|
166
|
+
if condition[1] == '${exports.REDIS_STORE_TOKEN.Missing}' then
|
|
167
|
+
if currentVersion ~= nil then return {'${exports.REDIS_STORE_TOKEN.Conflict}', nowMs} end
|
|
168
|
+
elseif condition[1] == '${exports.REDIS_STORE_TOKEN.Value}' then
|
|
169
|
+
if not record or record[2] ~= ARGV[condition[4]] then
|
|
170
|
+
return {'${exports.REDIS_STORE_TOKEN.Conflict}', nowMs}
|
|
171
|
+
end
|
|
122
172
|
elseif currentVersion ~= condition[4] then
|
|
123
|
-
return {'
|
|
173
|
+
return {'${exports.REDIS_STORE_TOKEN.Conflict}', nowMs}
|
|
124
174
|
end
|
|
125
175
|
end
|
|
126
176
|
|
|
127
177
|
for _, mutation in ipairs(mutations) do
|
|
128
178
|
local rowKey = KEYS[mutation[2] + 6]
|
|
129
|
-
|
|
130
|
-
|
|
131
|
-
|
|
132
|
-
if redis.call('ZCARD', rowKey) >= 128 then
|
|
133
|
-
return {'backlog', nowMs}
|
|
179
|
+
pruneHistory(rowKey, minimumBoundary)
|
|
180
|
+
if redis.call('ZCARD', rowKey) >= ${REDIS_ROW_COMPACTION_ENTRY_LIMIT} then
|
|
181
|
+
return {'${exports.REDIS_STORE_TOKEN.Backlog}', nowMs}
|
|
134
182
|
end
|
|
135
183
|
end
|
|
136
184
|
|
|
137
185
|
local sequence = tostring(redis.call('INCR', sequenceKey))
|
|
138
186
|
local byteArg = 3
|
|
139
|
-
|
|
187
|
+
for _, condition in ipairs(conditions) do
|
|
188
|
+
if condition[1] == '${exports.REDIS_STORE_TOKEN.Value}' then byteArg = byteArg + 1 end
|
|
189
|
+
end
|
|
190
|
+
local result = {'${exports.REDIS_STORE_TOKEN.Applied}', nowMs}
|
|
140
191
|
for _, mutation in ipairs(mutations) do
|
|
141
192
|
local rowKey = KEYS[mutation[2] + 6]
|
|
142
193
|
local originalKey = mutation[3]
|
|
143
|
-
if mutation[1] == '
|
|
194
|
+
if mutation[1] == '${exports.REDIS_STORE_TOKEN.Put}' then
|
|
144
195
|
local bytes = ARGV[byteArg]
|
|
145
196
|
byteArg = byteArg + 1
|
|
146
197
|
local retention = mutation[4]
|
|
@@ -156,105 +207,66 @@ for _, mutation in ipairs(mutations) do
|
|
|
156
207
|
redis.call('ZADD', indexKey, 0, originalKey)
|
|
157
208
|
redis.call('HSET', mapKey, originalKey, rowKey)
|
|
158
209
|
end
|
|
159
|
-
local dueAt = nowMs +
|
|
210
|
+
local dueAt = nowMs + ${REDIS_CLEANUP_RETRY_DELAY_MS}
|
|
160
211
|
local scheduled = redis.call('ZSCORE', cleanupKey, originalKey)
|
|
161
212
|
if not scheduled or tonumber(scheduled) > dueAt then
|
|
162
213
|
redis.call('ZADD', cleanupKey, dueAt, originalKey)
|
|
163
214
|
end
|
|
164
215
|
end
|
|
165
216
|
return result
|
|
166
|
-
|
|
217
|
+
`);
|
|
167
218
|
// Shared body for point-in-time paged scanning. KEYS[1]=indexKey,
|
|
168
219
|
// KEYS[2]=mapKey, KEYS[3]=snapshotKey, KEYS[4]=cleanupKey,
|
|
169
220
|
// KEYS[5]=sequenceKey, KEYS[6]=snapshotExpiryKey, KEYS[7]=snapshotBoundaryKey.
|
|
170
221
|
const SCAN_CLEANUP_AND_BOUNDARY = DECODE_HELPERS +
|
|
171
222
|
`
|
|
172
|
-
local
|
|
173
|
-
|
|
174
|
-
|
|
175
|
-
|
|
176
|
-
|
|
177
|
-
|
|
178
|
-
|
|
179
|
-
if #boundaryEntry == 2 then minimumBoundary = tonumber(boundaryEntry[2]) end
|
|
180
|
-
|
|
181
|
-
local due = redis.call('ZRANGEBYSCORE', KEYS[4], '-inf', nowMs, 'LIMIT', 0, 32)
|
|
182
|
-
for _, original in ipairs(due) do
|
|
183
|
-
local rowKey = redis.call('HGET', KEYS[2], original)
|
|
184
|
-
local members = {}
|
|
185
|
-
if rowKey then
|
|
186
|
-
members = redis.call('ZREVRANGE', rowKey, 0, 0, 'WITHSCORES')
|
|
187
|
-
end
|
|
188
|
-
if #members == 0 then
|
|
189
|
-
redis.call('ZREM', KEYS[1], original)
|
|
190
|
-
redis.call('HDEL', KEYS[2], original)
|
|
191
|
-
redis.call('ZREM', KEYS[4], original)
|
|
192
|
-
elseif minimumBoundary then
|
|
193
|
-
local anchor = redis.call('ZREVRANGEBYSCORE', rowKey, minimumBoundary, '-inf', 'WITHSCORES', 'LIMIT', 0, 1)
|
|
194
|
-
if #anchor == 2 then
|
|
195
|
-
redis.call('ZREMRANGEBYSCORE', rowKey, '-inf', '(' .. anchor[2])
|
|
196
|
-
end
|
|
197
|
-
redis.call('ZADD', KEYS[4], nowMs + 1000, original)
|
|
198
|
-
else
|
|
199
|
-
local record = decodeMember(members[1])
|
|
200
|
-
local expiresAtMs = tonumber(record[4])
|
|
201
|
-
if record[5] == true or (expiresAtMs > 0 and expiresAtMs + 60000 <= nowMs) then
|
|
202
|
-
redis.call('DEL', rowKey)
|
|
203
|
-
redis.call('ZREM', KEYS[1], original)
|
|
204
|
-
redis.call('HDEL', KEYS[2], original)
|
|
205
|
-
redis.call('ZREM', KEYS[4], original)
|
|
206
|
-
else
|
|
207
|
-
redis.call('ZREMRANGEBYRANK', rowKey, 0, -2)
|
|
208
|
-
if expiresAtMs > 0 then
|
|
209
|
-
redis.call('ZADD', KEYS[4], math.max(nowMs + 1000, expiresAtMs + 60000), original)
|
|
210
|
-
else
|
|
211
|
-
redis.call('ZREM', KEYS[4], original)
|
|
212
|
-
end
|
|
213
|
-
end
|
|
214
|
-
end
|
|
215
|
-
end
|
|
216
|
-
`;
|
|
223
|
+
local indexKey = KEYS[1]
|
|
224
|
+
local mapKey = KEYS[2]
|
|
225
|
+
local cleanupKey = KEYS[4]
|
|
226
|
+
local snapshotExpiryKey = KEYS[6]
|
|
227
|
+
local snapshotBoundaryKey = KEYS[7]
|
|
228
|
+
` +
|
|
229
|
+
CLEANUP_AND_BOUNDARY;
|
|
217
230
|
const SCAN_PAGE_READ = `
|
|
218
231
|
local metadata = redis.call('HMGET', KEYS[3], 'now', 'boundary', 'prefix')
|
|
219
232
|
if not metadata[1] or metadata[3] ~= prefix then
|
|
220
233
|
redis.call('ZREM', KEYS[6], snapshotId)
|
|
221
234
|
redis.call('ZREM', KEYS[7], snapshotId)
|
|
222
|
-
return {'
|
|
235
|
+
return {'${exports.REDIS_STORE_TOKEN.Expired}'}
|
|
223
236
|
end
|
|
224
237
|
local snapshotNow = tonumber(metadata[1])
|
|
225
238
|
local boundary = tonumber(metadata[2])
|
|
226
|
-
local lower = '
|
|
239
|
+
local lower = '[' .. prefix
|
|
240
|
+
local upper = '[' .. prefix .. string.char(${REDIS_UTF8_PREFIX_UPPER_BOUND_BYTE})
|
|
227
241
|
if string.len(lastKey) > 0 then lower = '(' .. lastKey end
|
|
228
|
-
local workLimit = math.max(limit *
|
|
229
|
-
local originals = redis.call('ZRANGEBYLEX', KEYS[1], lower,
|
|
242
|
+
local workLimit = math.max(limit * ${REDIS_SCAN_WORK_BUDGET_MULTIPLIER}, ${REDIS_SCAN_MINIMUM_WORK_BUDGET})
|
|
243
|
+
local originals = redis.call('ZRANGEBYLEX', KEYS[1], lower, upper, 'LIMIT', 0, workLimit + 1)
|
|
230
244
|
local emitted = 0
|
|
231
245
|
local encodedBytes = 0
|
|
232
246
|
local examined = 0
|
|
233
|
-
local result = {'
|
|
247
|
+
local result = {'${exports.REDIS_STORE_TOKEN.Page}', tostring(snapshotNow), ''}
|
|
234
248
|
while examined < #originals and examined < workLimit and emitted < limit do
|
|
235
249
|
local original = originals[examined + 1]
|
|
236
250
|
examined = examined + 1
|
|
237
|
-
|
|
238
|
-
|
|
239
|
-
|
|
240
|
-
|
|
241
|
-
|
|
242
|
-
|
|
243
|
-
|
|
244
|
-
|
|
245
|
-
|
|
246
|
-
|
|
247
|
-
|
|
248
|
-
|
|
249
|
-
break
|
|
250
|
-
end
|
|
251
|
-
table.insert(result, original)
|
|
252
|
-
table.insert(result, record[2])
|
|
253
|
-
table.insert(result, record[3])
|
|
254
|
-
table.insert(result, tostring(expiresAtMs))
|
|
255
|
-
encodedBytes = encodedBytes + itemBytes
|
|
256
|
-
emitted = emitted + 1
|
|
251
|
+
local rowKey = redis.call('HGET', KEYS[2], original)
|
|
252
|
+
if rowKey then
|
|
253
|
+
local members = redis.call('ZREVRANGEBYSCORE', rowKey, boundary, '-inf', 'LIMIT', 0, 1)
|
|
254
|
+
if #members > 0 then
|
|
255
|
+
local record = decodeMember(members[1])
|
|
256
|
+
local expiresAtMs = tonumber(record[4])
|
|
257
|
+
if record[1] == original and record[5] ~= true
|
|
258
|
+
and (expiresAtMs == 0 or expiresAtMs > snapshotNow) then
|
|
259
|
+
local itemBytes = string.len(original) + string.len(record[2]) + string.len(record[3]) + ${REDIS_ENCODED_ITEM_OVERHEAD_BYTES}
|
|
260
|
+
if emitted > 0 and encodedBytes + itemBytes > ${framework_internal_1.ZLINK_PROVIDER_MAX_ENCODED_PAGE_BYTES} then
|
|
261
|
+
examined = examined - 1
|
|
262
|
+
break
|
|
257
263
|
end
|
|
264
|
+
table.insert(result, original)
|
|
265
|
+
table.insert(result, record[2])
|
|
266
|
+
table.insert(result, record[3])
|
|
267
|
+
table.insert(result, tostring(expiresAtMs))
|
|
268
|
+
encodedBytes = encodedBytes + itemBytes
|
|
269
|
+
emitted = emitted + 1
|
|
258
270
|
end
|
|
259
271
|
end
|
|
260
272
|
end
|
|
@@ -280,14 +292,14 @@ local limit = tonumber(ARGV[2])
|
|
|
280
292
|
local snapshotId = ARGV[3]
|
|
281
293
|
local lastKey = ''
|
|
282
294
|
|
|
283
|
-
if redis.call('ZCARD', KEYS[6]) >=
|
|
284
|
-
return {'
|
|
295
|
+
if redis.call('ZCARD', KEYS[6]) >= ${REDIS_MAX_SCAN_SNAPSHOTS} then
|
|
296
|
+
return {'${exports.REDIS_STORE_TOKEN.Capacity}'}
|
|
285
297
|
end
|
|
286
298
|
redis.call('DEL', KEYS[3])
|
|
287
299
|
local boundary = tonumber(redis.call('GET', KEYS[5]) or '0')
|
|
288
300
|
redis.call('HSET', KEYS[3], 'now', tostring(nowMs), 'boundary', tostring(boundary), 'prefix', prefix)
|
|
289
|
-
redis.call('PEXPIRE', KEYS[3],
|
|
290
|
-
redis.call('ZADD', KEYS[6], nowMs +
|
|
301
|
+
redis.call('PEXPIRE', KEYS[3], ${REDIS_SCAN_SNAPSHOT_RETENTION_MS})
|
|
302
|
+
redis.call('ZADD', KEYS[6], nowMs + ${REDIS_SCAN_SNAPSHOT_RETENTION_MS}, snapshotId)
|
|
291
303
|
redis.call('ZADD', KEYS[7], boundary, snapshotId)
|
|
292
304
|
` +
|
|
293
305
|
SCAN_PAGE_READ;
|
|
@@ -303,7 +315,7 @@ local snapshotId = ARGV[4]
|
|
|
303
315
|
if redis.call('EXISTS', KEYS[3]) == 0 then
|
|
304
316
|
redis.call('ZREM', KEYS[6], snapshotId)
|
|
305
317
|
redis.call('ZREM', KEYS[7], snapshotId)
|
|
306
|
-
return {'
|
|
318
|
+
return {'${exports.REDIS_STORE_TOKEN.Expired}'}
|
|
307
319
|
end
|
|
308
320
|
` +
|
|
309
321
|
SCAN_PAGE_READ;
|
|
@@ -317,15 +329,15 @@ exports.BLOB_PUT_SCRIPT = PROLOGUE +
|
|
|
317
329
|
local existing = redis.call('GET', KEYS[1])
|
|
318
330
|
if existing then
|
|
319
331
|
if existing ~= ARGV[1] then
|
|
320
|
-
return {'
|
|
332
|
+
return {'${exports.REDIS_STORE_TOKEN.Conflict}', nowMs}
|
|
321
333
|
end
|
|
322
334
|
local ttl = redis.call('PTTL', KEYS[1])
|
|
323
335
|
local expiresAtMs = nowMs + math.max(ttl, 0)
|
|
324
|
-
return {'
|
|
336
|
+
return {'${exports.REDIS_STORE_TOKEN.AlreadyStored}', nowMs, tostring(expiresAtMs)}
|
|
325
337
|
end
|
|
326
338
|
local retentionMs = tonumber(ARGV[2])
|
|
327
339
|
redis.call('SET', KEYS[1], ARGV[1], 'PX', retentionMs)
|
|
328
|
-
return {'
|
|
340
|
+
return {'${exports.REDIS_STORE_TOKEN.Stored}', nowMs, tostring(nowMs + retentionMs)}
|
|
329
341
|
`;
|
|
330
342
|
exports.BLOB_READ_SCRIPT = PROLOGUE +
|
|
331
343
|
`
|
package/dist/opaque-store.js
CHANGED
|
@@ -1,13 +1,11 @@
|
|
|
1
1
|
"use strict";
|
|
2
2
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
3
|
exports.ZLinkRedisLocationStore = void 0;
|
|
4
|
+
const framework_internal_1 = require("./framework-internal");
|
|
4
5
|
const node_crypto_1 = require("node:crypto");
|
|
5
6
|
const redis_connection_1 = require("./redis-connection");
|
|
6
7
|
const opaque_redis_scripts_1 = require("./opaque-redis-scripts");
|
|
7
8
|
const redis_values_1 = require("./redis-values");
|
|
8
|
-
const MAX_VALUE_BYTES = 1024 * 1024;
|
|
9
|
-
const MAX_WRITE_KEYS = 2_048;
|
|
10
|
-
const MAX_WRITE_BYTES = 4 * 1024 * 1024;
|
|
11
9
|
// {prefix}:{zlink-location-v3}:opaque:{sha256hex(preimage)} is the public
|
|
12
10
|
// contract (21-location-runtime.md#2.4, 22-location-store-redis.md#7). The
|
|
13
11
|
// braces are a Redis Cluster hash tag: every key this provider's scripts
|
|
@@ -54,16 +52,17 @@ class ZLinkRedisLocationStore {
|
|
|
54
52
|
], [
|
|
55
53
|
JSON.stringify(encoded.conditions),
|
|
56
54
|
JSON.stringify(encoded.mutations),
|
|
55
|
+
...encoded.expectedBytes,
|
|
57
56
|
...encoded.putBytes
|
|
58
57
|
], signal));
|
|
59
58
|
const storeNow = fromUnixMs((0, redis_values_1.toNumber)(result[1]));
|
|
60
59
|
const outcome = (0, redis_values_1.asString)(result[0]);
|
|
61
|
-
if (outcome ===
|
|
60
|
+
if (outcome === opaque_redis_scripts_1.REDIS_STORE_TOKEN.Conflict)
|
|
62
61
|
return { kind: 'conflict', storeNow };
|
|
63
|
-
if (outcome ===
|
|
62
|
+
if (outcome === opaque_redis_scripts_1.REDIS_STORE_TOKEN.Backlog) {
|
|
64
63
|
throw new Error('Redis Location Store version backlog is full.');
|
|
65
64
|
}
|
|
66
|
-
if (outcome !==
|
|
65
|
+
if (outcome !== opaque_redis_scripts_1.REDIS_STORE_TOKEN.Applied) {
|
|
67
66
|
throw new Error('Redis Location Store returned an unrecognized write outcome.');
|
|
68
67
|
}
|
|
69
68
|
const putVersions = [];
|
|
@@ -82,6 +81,9 @@ class ZLinkRedisLocationStore {
|
|
|
82
81
|
return await this.readScanPage(snapshotId, await this.connection.eval(opaque_redis_scripts_1.OPAQUE_SCAN_START_SCRIPT, this.scanKeys(snapshotId), [request.prefix, String(request.limit), snapshotId], signal));
|
|
83
82
|
}
|
|
84
83
|
const cursor = parseCursor(request.cursor);
|
|
84
|
+
if (cursor === undefined) {
|
|
85
|
+
return { kind: 'expired' };
|
|
86
|
+
}
|
|
85
87
|
return await this.readScanPage(cursor.snapshotId, await this.connection.eval(opaque_redis_scripts_1.OPAQUE_SCAN_CONTINUE_SCRIPT, this.scanKeys(cursor.snapshotId), [request.prefix, cursor.lastKey, String(request.limit), cursor.snapshotId], signal));
|
|
86
88
|
}
|
|
87
89
|
async dispose() {
|
|
@@ -90,12 +92,12 @@ class ZLinkRedisLocationStore {
|
|
|
90
92
|
async readScanPage(snapshotId, raw) {
|
|
91
93
|
const result = (0, redis_values_1.asArray)(raw);
|
|
92
94
|
const outcome = (0, redis_values_1.asString)(result[0]);
|
|
93
|
-
if (outcome ===
|
|
95
|
+
if (outcome === opaque_redis_scripts_1.REDIS_STORE_TOKEN.Expired)
|
|
94
96
|
return { kind: 'expired' };
|
|
95
|
-
if (outcome ===
|
|
97
|
+
if (outcome === opaque_redis_scripts_1.REDIS_STORE_TOKEN.Capacity) {
|
|
96
98
|
throw new Error('Redis Location Store snapshot capacity is full.');
|
|
97
99
|
}
|
|
98
|
-
if (outcome !==
|
|
100
|
+
if (outcome !== opaque_redis_scripts_1.REDIS_STORE_TOKEN.Page) {
|
|
99
101
|
throw new Error('Redis Location Store returned an unrecognized scan outcome.');
|
|
100
102
|
}
|
|
101
103
|
const storeNow = fromUnixMs((0, redis_values_1.toNumber)(result[1]));
|
|
@@ -168,44 +170,53 @@ function encodeWrite(request) {
|
|
|
168
170
|
throw new RangeError('Location Store condition and mutation keys must be unique.');
|
|
169
171
|
}
|
|
170
172
|
const keys = [...new Set([...conditionKeys, ...mutationKeys])];
|
|
171
|
-
if (keys.length >
|
|
172
|
-
throw new RangeError(
|
|
173
|
+
if (keys.length > framework_internal_1.ZLINK_PROVIDER_MAX_WRITE_KEYS) {
|
|
174
|
+
throw new RangeError(`Location Store write exceeds ${framework_internal_1.ZLINK_PROVIDER_MAX_WRITE_KEYS.toLocaleString('en-US')} unique keys.`);
|
|
173
175
|
}
|
|
174
176
|
// Row keys are appended after the six fixed auxiliary keys; the script
|
|
175
177
|
// adds 6 to this 1-based index before indexing into KEYS.
|
|
176
178
|
const keyIndex = new Map(keys.map((key, index) => [key, index + 1]));
|
|
177
179
|
let encodedBytes = 0;
|
|
180
|
+
const expectedBytes = [];
|
|
178
181
|
const conditions = request.conditions.map((condition) => {
|
|
179
182
|
const key = requireKey(condition.key);
|
|
180
183
|
encodedBytes += Buffer.byteLength(key, 'utf8');
|
|
181
184
|
if (condition.kind === 'missing')
|
|
182
|
-
return [
|
|
185
|
+
return [opaque_redis_scripts_1.REDIS_STORE_TOKEN.Missing, keyIndex.get(key), key];
|
|
186
|
+
if (condition.kind === 'value') {
|
|
187
|
+
requireValue(condition.expected, undefined);
|
|
188
|
+
encodedBytes += condition.expected.byteLength;
|
|
189
|
+
expectedBytes.push(Buffer.from(condition.expected));
|
|
190
|
+
return [opaque_redis_scripts_1.REDIS_STORE_TOKEN.Value, keyIndex.get(key), key, expectedBytes.length + 2];
|
|
191
|
+
}
|
|
183
192
|
const expected = requireVersion(condition.expected);
|
|
184
193
|
encodedBytes += Buffer.byteLength(expected, 'utf8');
|
|
185
|
-
return [
|
|
194
|
+
return [opaque_redis_scripts_1.REDIS_STORE_TOKEN.Version, keyIndex.get(key), key, expected];
|
|
186
195
|
});
|
|
187
196
|
const putBytes = [];
|
|
188
197
|
const mutations = request.mutations.map((mutation) => {
|
|
189
198
|
const key = requireKey(mutation.key);
|
|
190
199
|
encodedBytes += Buffer.byteLength(key, 'utf8');
|
|
191
200
|
if (mutation.kind === 'delete')
|
|
192
|
-
return [
|
|
201
|
+
return [opaque_redis_scripts_1.REDIS_STORE_TOKEN.Delete, keyIndex.get(key), key];
|
|
193
202
|
const retentionMs = requireValue(mutation.bytes, mutation.retentionMs);
|
|
194
203
|
encodedBytes += mutation.bytes.byteLength;
|
|
195
204
|
putBytes.push(Buffer.from(mutation.bytes));
|
|
196
|
-
return [
|
|
205
|
+
return [opaque_redis_scripts_1.REDIS_STORE_TOKEN.Put, keyIndex.get(key), key, retentionMs ?? false];
|
|
197
206
|
});
|
|
198
|
-
if (encodedBytes >
|
|
199
|
-
throw new RangeError(
|
|
207
|
+
if (encodedBytes > framework_internal_1.ZLINK_PROVIDER_MAX_WRITE_BYTES) {
|
|
208
|
+
throw new RangeError(`Location Store write exceeds ${framework_internal_1.ZLINK_PROVIDER_MAX_WRITE_BYTES / (1024 * 1024)} MiB encoded input.`);
|
|
200
209
|
}
|
|
201
|
-
return { keys, conditions, mutations, putBytes };
|
|
210
|
+
return { keys, conditions, mutations, expectedBytes, putBytes };
|
|
202
211
|
}
|
|
203
212
|
function requireScanRequest(request) {
|
|
204
|
-
if (Buffer.byteLength(request.prefix, 'utf8') >
|
|
205
|
-
throw new RangeError(
|
|
213
|
+
if (Buffer.byteLength(request.prefix, 'utf8') > framework_internal_1.ZLINK_PROVIDER_MAX_KEY_BYTES) {
|
|
214
|
+
throw new RangeError(`Location Store scan prefix exceeds ${framework_internal_1.ZLINK_PROVIDER_MAX_KEY_BYTES.toLocaleString('en-US')} UTF-8 bytes.`);
|
|
206
215
|
}
|
|
207
|
-
if (!Number.isSafeInteger(request.limit) ||
|
|
208
|
-
|
|
216
|
+
if (!Number.isSafeInteger(request.limit) ||
|
|
217
|
+
request.limit < 1 ||
|
|
218
|
+
request.limit > framework_internal_1.ZLINK_PROVIDER_MAX_PAGE_SIZE) {
|
|
219
|
+
throw new RangeError(`Location Store scan limit must be in 1..${framework_internal_1.ZLINK_PROVIDER_MAX_PAGE_SIZE}.`);
|
|
209
220
|
}
|
|
210
221
|
if (request.cursor !== undefined)
|
|
211
222
|
requireCursor(request.cursor);
|
|
@@ -213,8 +224,8 @@ function requireScanRequest(request) {
|
|
|
213
224
|
function requireKey(key) {
|
|
214
225
|
const value = key.value;
|
|
215
226
|
const bytes = Buffer.byteLength(value, 'utf8');
|
|
216
|
-
if (bytes < 1 || bytes >
|
|
217
|
-
throw new RangeError(
|
|
227
|
+
if (bytes < 1 || bytes > framework_internal_1.ZLINK_PROVIDER_MAX_KEY_BYTES) {
|
|
228
|
+
throw new RangeError(`Location Store key must contain 1..${framework_internal_1.ZLINK_PROVIDER_MAX_KEY_BYTES.toLocaleString('en-US')} UTF-8 bytes.`);
|
|
218
229
|
}
|
|
219
230
|
return value;
|
|
220
231
|
}
|
|
@@ -226,22 +237,22 @@ function requireMatchingKey(actual, expected) {
|
|
|
226
237
|
function requireVersion(version) {
|
|
227
238
|
const value = version.value;
|
|
228
239
|
const bytes = Buffer.byteLength(value, 'utf8');
|
|
229
|
-
if (bytes < 1 || bytes >
|
|
230
|
-
throw new RangeError(
|
|
240
|
+
if (bytes < 1 || bytes > framework_internal_1.ZLINK_PROVIDER_MAX_VERSION_BYTES) {
|
|
241
|
+
throw new RangeError(`Location Store version must contain 1..${framework_internal_1.ZLINK_PROVIDER_MAX_VERSION_BYTES.toLocaleString('en-US')} UTF-8 bytes.`);
|
|
231
242
|
}
|
|
232
243
|
return value;
|
|
233
244
|
}
|
|
234
245
|
function requireCursor(cursor) {
|
|
235
246
|
const value = cursor.value;
|
|
236
247
|
const bytes = Buffer.byteLength(value, 'utf8');
|
|
237
|
-
if (bytes < 1 || bytes >
|
|
238
|
-
throw new RangeError(
|
|
248
|
+
if (bytes < 1 || bytes > framework_internal_1.ZLINK_PROVIDER_MAX_SCAN_CURSOR_BYTES) {
|
|
249
|
+
throw new RangeError(`Location Store cursor must contain 1..${framework_internal_1.ZLINK_PROVIDER_MAX_SCAN_CURSOR_BYTES.toLocaleString('en-US')} UTF-8 bytes.`);
|
|
239
250
|
}
|
|
240
251
|
return value;
|
|
241
252
|
}
|
|
242
253
|
function requireValue(bytes, retentionMs) {
|
|
243
|
-
if (bytes.byteLength >
|
|
244
|
-
throw new RangeError(
|
|
254
|
+
if (bytes.byteLength > framework_internal_1.ZLINK_PROVIDER_MAX_VALUE_BYTES) {
|
|
255
|
+
throw new RangeError(`Location Store value exceeds ${framework_internal_1.ZLINK_PROVIDER_MAX_VALUE_BYTES / (1024 * 1024)} MiB.`);
|
|
245
256
|
}
|
|
246
257
|
if (retentionMs !== undefined &&
|
|
247
258
|
(!Number.isFinite(retentionMs) ||
|
|
@@ -252,14 +263,14 @@ function requireValue(bytes, retentionMs) {
|
|
|
252
263
|
return retentionMs === undefined ? undefined : Math.ceil(retentionMs);
|
|
253
264
|
}
|
|
254
265
|
function parseCursor(cursor) {
|
|
255
|
-
const value =
|
|
266
|
+
const value = cursor.value;
|
|
256
267
|
const separator = value.lastIndexOf(':');
|
|
257
268
|
const snapshotId = separator < 0 ? '' : value.slice(0, separator);
|
|
258
269
|
const lastKeyHex = separator < 0 ? '' : value.slice(separator + 1);
|
|
259
270
|
if (!/^[0-9a-f-]{36}$/.test(snapshotId) ||
|
|
260
271
|
!/^[0-9a-f]*$/.test(lastKeyHex) ||
|
|
261
272
|
lastKeyHex.length % 2 !== 0) {
|
|
262
|
-
|
|
273
|
+
return undefined;
|
|
263
274
|
}
|
|
264
275
|
return { snapshotId, lastKey: Buffer.from(lastKeyHex, 'hex').toString('utf8') };
|
|
265
276
|
}
|
|
@@ -10,7 +10,7 @@ export declare class RedisConnection {
|
|
|
10
10
|
private readonly operationTimeoutMs?;
|
|
11
11
|
constructor(options: ZLinkRedisLocationOptions | ZLinkRedisRelocationOptions);
|
|
12
12
|
command(args: RedisCommandValue[], signal?: AbortSignal): Promise<unknown>;
|
|
13
|
-
eval(script:
|
|
13
|
+
eval(script: RedisCommandValue, keys: readonly string[], args: readonly RedisCommandValue[], signal?: AbortSignal): Promise<unknown>;
|
|
14
14
|
dispose(): Promise<void>;
|
|
15
15
|
private connected;
|
|
16
16
|
}
|