@zlink-systems/framework-locations-redis 0.25.0 → 0.27.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/.tsbuildinfo +1 -1
- package/dist/framework-internal.d.ts +1 -0
- package/dist/framework-internal.js +12 -0
- package/dist/opaque-redis-scripts.d.ts +17 -2
- package/dist/opaque-redis-scripts.js +126 -114
- package/dist/opaque-store.js +50 -35
- package/dist/redis-connection.d.ts +1 -1
- package/dist/relocation-store.js +13 -10
- package/package.json +3 -3
- package/src/framework-internal.ts +20 -0
- package/src/opaque-redis-scripts.ts +135 -116
- package/src/opaque-store.ts +87 -41
- package/src/redis-connection.ts +1 -1
- package/src/relocation-store.ts +24 -10
package/dist/relocation-store.js
CHANGED
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
"use strict";
|
|
2
2
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
3
|
exports.ZLinkRedisRelocationStore = void 0;
|
|
4
|
+
const framework_internal_1 = require("./framework-internal");
|
|
4
5
|
const redis_connection_1 = require("./redis-connection");
|
|
5
6
|
const opaque_redis_scripts_1 = require("./opaque-redis-scripts");
|
|
6
7
|
const redis_values_1 = require("./redis-values");
|
|
@@ -21,12 +22,12 @@ class ZLinkRedisRelocationStore {
|
|
|
21
22
|
requirePayload(payload);
|
|
22
23
|
const retention = requireRetention(retentionMs);
|
|
23
24
|
const result = (0, redis_values_1.asArray)(await this.connection.eval(opaque_redis_scripts_1.BLOB_PUT_SCRIPT, [this.blobKey(referenceValue)], [Buffer.from(payload), String(retention)], signal));
|
|
24
|
-
const
|
|
25
|
+
const outcome = (0, redis_values_1.asString)(result[0]);
|
|
25
26
|
const storeNow = fromUnixMs((0, redis_values_1.toNumber)(result[1]));
|
|
26
|
-
if (
|
|
27
|
-
return { kind, storeNow };
|
|
27
|
+
if (outcome === opaque_redis_scripts_1.REDIS_STORE_TOKEN.Conflict)
|
|
28
|
+
return { kind: 'conflict', storeNow };
|
|
28
29
|
return {
|
|
29
|
-
kind:
|
|
30
|
+
kind: outcome === opaque_redis_scripts_1.REDIS_STORE_TOKEN.AlreadyStored ? 'alreadyStored' : 'stored',
|
|
30
31
|
storeNow,
|
|
31
32
|
expiresAt: fromUnixMs((0, redis_values_1.toNumber)(result[2]))
|
|
32
33
|
};
|
|
@@ -72,21 +73,23 @@ exports.ZLinkRedisRelocationStore = ZLinkRedisRelocationStore;
|
|
|
72
73
|
function requireReference(reference) {
|
|
73
74
|
const value = reference.value;
|
|
74
75
|
const bytes = Buffer.byteLength(value, 'utf8');
|
|
75
|
-
if (bytes < 1 || bytes >
|
|
76
|
-
throw new RangeError(
|
|
76
|
+
if (bytes < 1 || bytes > framework_internal_1.ZLINK_RELOCATION_MAX_BLOB_REFERENCE_BYTES) {
|
|
77
|
+
throw new RangeError(`Relocation Store reference must contain 1..${framework_internal_1.ZLINK_RELOCATION_MAX_BLOB_REFERENCE_BYTES.toLocaleString('en-US')} UTF-8 bytes.`);
|
|
77
78
|
}
|
|
78
79
|
return value;
|
|
79
80
|
}
|
|
80
81
|
function requirePayload(payload) {
|
|
81
82
|
if (payload.byteLength > MAX_ENCODED_BLOB_BYTES) {
|
|
82
|
-
throw new RangeError(
|
|
83
|
+
throw new RangeError(`Relocation Store encoded blob exceeds ${Math.floor(MAX_ENCODED_BLOB_BYTES / (1024 * 1024))} MiB + ${MAX_ENCODED_BLOB_BYTES % (1024 * 1024)} bytes.`);
|
|
83
84
|
}
|
|
84
85
|
}
|
|
85
86
|
function requireRetention(retentionMs) {
|
|
86
|
-
if (!Number.
|
|
87
|
-
|
|
87
|
+
if (!Number.isFinite(retentionMs) ||
|
|
88
|
+
retentionMs <= 0 ||
|
|
89
|
+
!Number.isSafeInteger(Math.ceil(retentionMs))) {
|
|
90
|
+
throw new RangeError('Relocation Store retention must round to a positive safe integer.');
|
|
88
91
|
}
|
|
89
|
-
return retentionMs;
|
|
92
|
+
return Math.ceil(retentionMs);
|
|
90
93
|
}
|
|
91
94
|
function asBuffer(value) {
|
|
92
95
|
if (Buffer.isBuffer(value))
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@zlink-systems/framework-locations-redis",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.27.0",
|
|
4
4
|
"license": "SEE LICENSE IN LICENSE",
|
|
5
5
|
"repository": {
|
|
6
6
|
"type": "git",
|
|
@@ -17,8 +17,8 @@
|
|
|
17
17
|
}
|
|
18
18
|
},
|
|
19
19
|
"dependencies": {
|
|
20
|
-
"@zlink-systems/framework": "0.
|
|
21
|
-
"@zlink-systems/zlink": "1.
|
|
20
|
+
"@zlink-systems/framework": "0.27.0",
|
|
21
|
+
"@zlink-systems/zlink": "1.14.0",
|
|
22
22
|
"redis": "^6.1.0"
|
|
23
23
|
}
|
|
24
24
|
}
|
|
@@ -0,0 +1,20 @@
|
|
|
1
|
+
import { createRequire } from 'node:module';
|
|
2
|
+
import path from 'node:path';
|
|
3
|
+
|
|
4
|
+
const requireFramework = createRequire(__filename);
|
|
5
|
+
const frameworkEntry = requireFramework.resolve('@zlink-systems/framework');
|
|
6
|
+
const internal = requireFramework(
|
|
7
|
+
path.join(path.dirname(frameworkEntry), 'internal')
|
|
8
|
+
) as typeof import('../../framework/src/internal');
|
|
9
|
+
|
|
10
|
+
export const {
|
|
11
|
+
ZLINK_PROVIDER_MAX_KEY_BYTES,
|
|
12
|
+
ZLINK_PROVIDER_MAX_VALUE_BYTES,
|
|
13
|
+
ZLINK_PROVIDER_MAX_VERSION_BYTES,
|
|
14
|
+
ZLINK_PROVIDER_MAX_WRITE_KEYS,
|
|
15
|
+
ZLINK_PROVIDER_MAX_WRITE_BYTES,
|
|
16
|
+
ZLINK_PROVIDER_MAX_ENCODED_PAGE_BYTES,
|
|
17
|
+
ZLINK_PROVIDER_MAX_SCAN_CURSOR_BYTES,
|
|
18
|
+
ZLINK_PROVIDER_MAX_PAGE_SIZE,
|
|
19
|
+
ZLINK_RELOCATION_MAX_BLOB_REFERENCE_BYTES
|
|
20
|
+
} = internal;
|
|
@@ -1,7 +1,38 @@
|
|
|
1
|
+
import { ZLINK_PROVIDER_MAX_ENCODED_PAGE_BYTES } from './framework-internal';
|
|
2
|
+
const REDIS_MILLISECONDS_PER_SECOND = 1000;
|
|
3
|
+
const REDIS_MICROSECONDS_PER_MILLISECOND = 1000;
|
|
4
|
+
const REDIS_CLEANUP_RETRY_DELAY_MS = 1000;
|
|
5
|
+
const REDIS_EXPIRED_SNAPSHOT_BATCH_SIZE = 128;
|
|
6
|
+
const REDIS_ROW_COMPACTION_ENTRY_LIMIT = 128;
|
|
7
|
+
const REDIS_SCAN_MINIMUM_WORK_BUDGET = 128;
|
|
8
|
+
const REDIS_ENCODED_ITEM_OVERHEAD_BYTES = 128;
|
|
9
|
+
const REDIS_SCAN_WORK_BUDGET_MULTIPLIER = 4;
|
|
10
|
+
const REDIS_CLEANUP_BATCH_SIZE = 32;
|
|
11
|
+
const REDIS_RECORD_RETENTION_MS = 60000;
|
|
12
|
+
const REDIS_SCAN_SNAPSHOT_RETENTION_MS = 60000;
|
|
13
|
+
const REDIS_MAX_SCAN_SNAPSHOTS = 4096;
|
|
14
|
+
const REDIS_UTF8_PREFIX_UPPER_BOUND_BYTE = 0xff;
|
|
15
|
+
|
|
16
|
+
export const REDIS_STORE_TOKEN = Object.freeze({
|
|
17
|
+
Conflict: 'conflict',
|
|
18
|
+
Backlog: 'backlog',
|
|
19
|
+
Applied: 'applied',
|
|
20
|
+
Expired: 'expired',
|
|
21
|
+
Capacity: 'capacity',
|
|
22
|
+
Page: 'page',
|
|
23
|
+
AlreadyStored: 'alreadyStored',
|
|
24
|
+
Stored: 'stored',
|
|
25
|
+
Missing: 'missing',
|
|
26
|
+
Value: 'value',
|
|
27
|
+
Version: 'version',
|
|
28
|
+
Put: 'put',
|
|
29
|
+
Delete: 'delete'
|
|
30
|
+
} as const);
|
|
31
|
+
|
|
1
32
|
const PROLOGUE = `
|
|
2
33
|
if redis.replicate_commands then redis.replicate_commands() end
|
|
3
34
|
local time = redis.call('TIME')
|
|
4
|
-
local nowMs = tonumber(time[1]) *
|
|
35
|
+
local nowMs = tonumber(time[1]) * ${REDIS_MILLISECONDS_PER_SECOND} + math.floor(tonumber(time[2]) / ${REDIS_MICROSECONDS_PER_MILLISECOND})
|
|
5
36
|
`;
|
|
6
37
|
|
|
7
38
|
// Shared decode helper. Every opaque row is a Redis ZSET append-log; the
|
|
@@ -38,14 +69,15 @@ local function encodeMember(originalKey, bytes, version, expiresAtMs, tombstone)
|
|
|
38
69
|
end
|
|
39
70
|
`;
|
|
40
71
|
|
|
41
|
-
export const OPAQUE_READ_SCRIPT =
|
|
72
|
+
export const OPAQUE_READ_SCRIPT = Buffer.from(
|
|
42
73
|
PROLOGUE +
|
|
43
|
-
|
|
44
|
-
|
|
74
|
+
DECODE_HELPERS +
|
|
75
|
+
`
|
|
45
76
|
local record = liveRecordAt(KEYS[1], nowMs)
|
|
46
77
|
if not record then return {0, nowMs} end
|
|
47
78
|
return {1, nowMs, record[1], record[2], record[3], tostring(tonumber(record[4]))}
|
|
48
|
-
|
|
79
|
+
`
|
|
80
|
+
);
|
|
49
81
|
|
|
50
82
|
// KEYS[1..6] = indexKey, mapKey, cleanupKey, sequenceKey, snapshotExpiryKey,
|
|
51
83
|
// snapshotBoundaryKey (private auxiliary keys, not part of the public
|
|
@@ -53,22 +85,29 @@ return {1, nowMs, record[1], record[2], record[3], tostring(tonumber(record[4]))
|
|
|
53
85
|
// conditions/mutations, in the order fixed by ARGV[1]/ARGV[2].
|
|
54
86
|
// ARGV[1] = JSON conditions: ['missing', keyIndex, originalKey]
|
|
55
87
|
// | ['version', keyIndex, originalKey, expectedVersion]
|
|
88
|
+
// | ['value', keyIndex, originalKey, expectedBytesArgIndex]
|
|
56
89
|
// ARGV[2] = JSON mutation metadata (no raw bytes, JSON must stay text-safe):
|
|
57
90
|
// ['put', keyIndex, originalKey, retentionMsOrFalse]
|
|
58
91
|
// | ['delete', keyIndex, originalKey]
|
|
59
|
-
// ARGV[3..] =
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
local
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
|
|
92
|
+
// ARGV[3..] = raw expected bytes for 'value' conditions, then raw bytes for
|
|
93
|
+
// 'put' mutations, each in request order.
|
|
94
|
+
const PRUNE_HISTORY_HELPER = `
|
|
95
|
+
local function pruneHistory(rowKey, minimumBoundary)
|
|
96
|
+
if minimumBoundary then
|
|
97
|
+
local anchor = redis.call('ZREVRANGEBYSCORE', rowKey, minimumBoundary, '-inf', 'WITHSCORES', 'LIMIT', 0, 1)
|
|
98
|
+
if #anchor == 2 then
|
|
99
|
+
redis.call('ZREMRANGEBYSCORE', rowKey, '-inf', '(' .. anchor[2])
|
|
100
|
+
end
|
|
101
|
+
else
|
|
102
|
+
redis.call('ZREMRANGEBYRANK', rowKey, 0, -2)
|
|
103
|
+
end
|
|
104
|
+
end
|
|
105
|
+
`;
|
|
70
106
|
|
|
71
|
-
|
|
107
|
+
const CLEANUP_AND_BOUNDARY =
|
|
108
|
+
PRUNE_HISTORY_HELPER +
|
|
109
|
+
`
|
|
110
|
+
local expiredSnapshots = redis.call('ZRANGEBYSCORE', snapshotExpiryKey, '-inf', nowMs, 'LIMIT', 0, ${REDIS_EXPIRED_SNAPSHOT_BATCH_SIZE})
|
|
72
111
|
for _, snapshotId in ipairs(expiredSnapshots) do
|
|
73
112
|
redis.call('ZREM', snapshotExpiryKey, snapshotId)
|
|
74
113
|
redis.call('ZREM', snapshotBoundaryKey, snapshotId)
|
|
@@ -77,7 +116,7 @@ local minimumBoundary = nil
|
|
|
77
116
|
local boundaryEntry = redis.call('ZRANGE', snapshotBoundaryKey, 0, 0, 'WITHSCORES')
|
|
78
117
|
if #boundaryEntry == 2 then minimumBoundary = tonumber(boundaryEntry[2]) end
|
|
79
118
|
|
|
80
|
-
local due = redis.call('ZRANGEBYSCORE', cleanupKey, '-inf', nowMs, 'LIMIT', 0,
|
|
119
|
+
local due = redis.call('ZRANGEBYSCORE', cleanupKey, '-inf', nowMs, 'LIMIT', 0, ${REDIS_CLEANUP_BATCH_SIZE})
|
|
81
120
|
for _, original in ipairs(due) do
|
|
82
121
|
local rowKey = redis.call('HGET', mapKey, original)
|
|
83
122
|
local members = {}
|
|
@@ -89,23 +128,20 @@ for _, original in ipairs(due) do
|
|
|
89
128
|
redis.call('HDEL', mapKey, original)
|
|
90
129
|
redis.call('ZREM', cleanupKey, original)
|
|
91
130
|
elseif minimumBoundary then
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
redis.call('ZREMRANGEBYSCORE', rowKey, '-inf', '(' .. anchor[2])
|
|
95
|
-
end
|
|
96
|
-
redis.call('ZADD', cleanupKey, nowMs + 1000, original)
|
|
131
|
+
pruneHistory(rowKey, minimumBoundary)
|
|
132
|
+
redis.call('ZADD', cleanupKey, nowMs + ${REDIS_CLEANUP_RETRY_DELAY_MS}, original)
|
|
97
133
|
else
|
|
98
134
|
local record = decodeMember(members[1])
|
|
99
135
|
local expiresAtMs = tonumber(record[4])
|
|
100
|
-
if record[5] == true or (expiresAtMs > 0 and expiresAtMs +
|
|
136
|
+
if record[5] == true or (expiresAtMs > 0 and expiresAtMs + ${REDIS_RECORD_RETENTION_MS} <= nowMs) then
|
|
101
137
|
redis.call('DEL', rowKey)
|
|
102
138
|
redis.call('ZREM', indexKey, original)
|
|
103
139
|
redis.call('HDEL', mapKey, original)
|
|
104
140
|
redis.call('ZREM', cleanupKey, original)
|
|
105
141
|
else
|
|
106
|
-
|
|
142
|
+
pruneHistory(rowKey, minimumBoundary)
|
|
107
143
|
if expiresAtMs > 0 then
|
|
108
|
-
redis.call('ZADD', cleanupKey, math.max(nowMs +
|
|
144
|
+
redis.call('ZADD', cleanupKey, math.max(nowMs + ${REDIS_CLEANUP_RETRY_DELAY_MS}, expiresAtMs + ${REDIS_RECORD_RETENTION_MS}), original)
|
|
109
145
|
else
|
|
110
146
|
redis.call('ZREM', cleanupKey, original)
|
|
111
147
|
end
|
|
@@ -113,36 +149,57 @@ for _, original in ipairs(due) do
|
|
|
113
149
|
end
|
|
114
150
|
end
|
|
115
151
|
|
|
152
|
+
`;
|
|
153
|
+
|
|
154
|
+
export const OPAQUE_WRITE_SCRIPT = Buffer.from(
|
|
155
|
+
PROLOGUE +
|
|
156
|
+
DECODE_HELPERS +
|
|
157
|
+
`
|
|
158
|
+
local indexKey = KEYS[1]
|
|
159
|
+
local mapKey = KEYS[2]
|
|
160
|
+
local cleanupKey = KEYS[3]
|
|
161
|
+
local sequenceKey = KEYS[4]
|
|
162
|
+
local snapshotExpiryKey = KEYS[5]
|
|
163
|
+
local snapshotBoundaryKey = KEYS[6]
|
|
164
|
+
|
|
165
|
+
` +
|
|
166
|
+
CLEANUP_AND_BOUNDARY +
|
|
167
|
+
`
|
|
116
168
|
local conditions = cjson.decode(ARGV[1])
|
|
117
169
|
local mutations = cjson.decode(ARGV[2])
|
|
118
170
|
|
|
119
171
|
for _, condition in ipairs(conditions) do
|
|
120
172
|
local record = liveRecordAt(KEYS[condition[2] + 6], nowMs)
|
|
121
173
|
local currentVersion = record and record[3] or nil
|
|
122
|
-
if condition[1] == '
|
|
123
|
-
if currentVersion ~= nil then return {'
|
|
174
|
+
if condition[1] == '${REDIS_STORE_TOKEN.Missing}' then
|
|
175
|
+
if currentVersion ~= nil then return {'${REDIS_STORE_TOKEN.Conflict}', nowMs} end
|
|
176
|
+
elseif condition[1] == '${REDIS_STORE_TOKEN.Value}' then
|
|
177
|
+
if not record or record[2] ~= ARGV[condition[4]] then
|
|
178
|
+
return {'${REDIS_STORE_TOKEN.Conflict}', nowMs}
|
|
179
|
+
end
|
|
124
180
|
elseif currentVersion ~= condition[4] then
|
|
125
|
-
return {'
|
|
181
|
+
return {'${REDIS_STORE_TOKEN.Conflict}', nowMs}
|
|
126
182
|
end
|
|
127
183
|
end
|
|
128
184
|
|
|
129
185
|
for _, mutation in ipairs(mutations) do
|
|
130
186
|
local rowKey = KEYS[mutation[2] + 6]
|
|
131
|
-
|
|
132
|
-
|
|
133
|
-
|
|
134
|
-
if redis.call('ZCARD', rowKey) >= 128 then
|
|
135
|
-
return {'backlog', nowMs}
|
|
187
|
+
pruneHistory(rowKey, minimumBoundary)
|
|
188
|
+
if redis.call('ZCARD', rowKey) >= ${REDIS_ROW_COMPACTION_ENTRY_LIMIT} then
|
|
189
|
+
return {'${REDIS_STORE_TOKEN.Backlog}', nowMs}
|
|
136
190
|
end
|
|
137
191
|
end
|
|
138
192
|
|
|
139
193
|
local sequence = tostring(redis.call('INCR', sequenceKey))
|
|
140
194
|
local byteArg = 3
|
|
141
|
-
|
|
195
|
+
for _, condition in ipairs(conditions) do
|
|
196
|
+
if condition[1] == '${REDIS_STORE_TOKEN.Value}' then byteArg = byteArg + 1 end
|
|
197
|
+
end
|
|
198
|
+
local result = {'${REDIS_STORE_TOKEN.Applied}', nowMs}
|
|
142
199
|
for _, mutation in ipairs(mutations) do
|
|
143
200
|
local rowKey = KEYS[mutation[2] + 6]
|
|
144
201
|
local originalKey = mutation[3]
|
|
145
|
-
if mutation[1] == '
|
|
202
|
+
if mutation[1] == '${REDIS_STORE_TOKEN.Put}' then
|
|
146
203
|
local bytes = ARGV[byteArg]
|
|
147
204
|
byteArg = byteArg + 1
|
|
148
205
|
local retention = mutation[4]
|
|
@@ -158,14 +215,15 @@ for _, mutation in ipairs(mutations) do
|
|
|
158
215
|
redis.call('ZADD', indexKey, 0, originalKey)
|
|
159
216
|
redis.call('HSET', mapKey, originalKey, rowKey)
|
|
160
217
|
end
|
|
161
|
-
local dueAt = nowMs +
|
|
218
|
+
local dueAt = nowMs + ${REDIS_CLEANUP_RETRY_DELAY_MS}
|
|
162
219
|
local scheduled = redis.call('ZSCORE', cleanupKey, originalKey)
|
|
163
220
|
if not scheduled or tonumber(scheduled) > dueAt then
|
|
164
221
|
redis.call('ZADD', cleanupKey, dueAt, originalKey)
|
|
165
222
|
end
|
|
166
223
|
end
|
|
167
224
|
return result
|
|
168
|
-
|
|
225
|
+
`
|
|
226
|
+
);
|
|
169
227
|
|
|
170
228
|
// Shared body for point-in-time paged scanning. KEYS[1]=indexKey,
|
|
171
229
|
// KEYS[2]=mapKey, KEYS[3]=snapshotKey, KEYS[4]=cleanupKey,
|
|
@@ -173,93 +231,54 @@ return result
|
|
|
173
231
|
const SCAN_CLEANUP_AND_BOUNDARY =
|
|
174
232
|
DECODE_HELPERS +
|
|
175
233
|
`
|
|
176
|
-
local
|
|
177
|
-
|
|
178
|
-
|
|
179
|
-
|
|
180
|
-
|
|
181
|
-
|
|
182
|
-
|
|
183
|
-
if #boundaryEntry == 2 then minimumBoundary = tonumber(boundaryEntry[2]) end
|
|
184
|
-
|
|
185
|
-
local due = redis.call('ZRANGEBYSCORE', KEYS[4], '-inf', nowMs, 'LIMIT', 0, 32)
|
|
186
|
-
for _, original in ipairs(due) do
|
|
187
|
-
local rowKey = redis.call('HGET', KEYS[2], original)
|
|
188
|
-
local members = {}
|
|
189
|
-
if rowKey then
|
|
190
|
-
members = redis.call('ZREVRANGE', rowKey, 0, 0, 'WITHSCORES')
|
|
191
|
-
end
|
|
192
|
-
if #members == 0 then
|
|
193
|
-
redis.call('ZREM', KEYS[1], original)
|
|
194
|
-
redis.call('HDEL', KEYS[2], original)
|
|
195
|
-
redis.call('ZREM', KEYS[4], original)
|
|
196
|
-
elseif minimumBoundary then
|
|
197
|
-
local anchor = redis.call('ZREVRANGEBYSCORE', rowKey, minimumBoundary, '-inf', 'WITHSCORES', 'LIMIT', 0, 1)
|
|
198
|
-
if #anchor == 2 then
|
|
199
|
-
redis.call('ZREMRANGEBYSCORE', rowKey, '-inf', '(' .. anchor[2])
|
|
200
|
-
end
|
|
201
|
-
redis.call('ZADD', KEYS[4], nowMs + 1000, original)
|
|
202
|
-
else
|
|
203
|
-
local record = decodeMember(members[1])
|
|
204
|
-
local expiresAtMs = tonumber(record[4])
|
|
205
|
-
if record[5] == true or (expiresAtMs > 0 and expiresAtMs + 60000 <= nowMs) then
|
|
206
|
-
redis.call('DEL', rowKey)
|
|
207
|
-
redis.call('ZREM', KEYS[1], original)
|
|
208
|
-
redis.call('HDEL', KEYS[2], original)
|
|
209
|
-
redis.call('ZREM', KEYS[4], original)
|
|
210
|
-
else
|
|
211
|
-
redis.call('ZREMRANGEBYRANK', rowKey, 0, -2)
|
|
212
|
-
if expiresAtMs > 0 then
|
|
213
|
-
redis.call('ZADD', KEYS[4], math.max(nowMs + 1000, expiresAtMs + 60000), original)
|
|
214
|
-
else
|
|
215
|
-
redis.call('ZREM', KEYS[4], original)
|
|
216
|
-
end
|
|
217
|
-
end
|
|
218
|
-
end
|
|
219
|
-
end
|
|
220
|
-
`;
|
|
234
|
+
local indexKey = KEYS[1]
|
|
235
|
+
local mapKey = KEYS[2]
|
|
236
|
+
local cleanupKey = KEYS[4]
|
|
237
|
+
local snapshotExpiryKey = KEYS[6]
|
|
238
|
+
local snapshotBoundaryKey = KEYS[7]
|
|
239
|
+
` +
|
|
240
|
+
CLEANUP_AND_BOUNDARY;
|
|
221
241
|
|
|
222
242
|
const SCAN_PAGE_READ = `
|
|
223
243
|
local metadata = redis.call('HMGET', KEYS[3], 'now', 'boundary', 'prefix')
|
|
224
244
|
if not metadata[1] or metadata[3] ~= prefix then
|
|
225
245
|
redis.call('ZREM', KEYS[6], snapshotId)
|
|
226
246
|
redis.call('ZREM', KEYS[7], snapshotId)
|
|
227
|
-
return {'
|
|
247
|
+
return {'${REDIS_STORE_TOKEN.Expired}'}
|
|
228
248
|
end
|
|
229
249
|
local snapshotNow = tonumber(metadata[1])
|
|
230
250
|
local boundary = tonumber(metadata[2])
|
|
231
|
-
local lower = '
|
|
251
|
+
local lower = '[' .. prefix
|
|
252
|
+
local upper = '[' .. prefix .. string.char(${REDIS_UTF8_PREFIX_UPPER_BOUND_BYTE})
|
|
232
253
|
if string.len(lastKey) > 0 then lower = '(' .. lastKey end
|
|
233
|
-
local workLimit = math.max(limit *
|
|
234
|
-
local originals = redis.call('ZRANGEBYLEX', KEYS[1], lower,
|
|
254
|
+
local workLimit = math.max(limit * ${REDIS_SCAN_WORK_BUDGET_MULTIPLIER}, ${REDIS_SCAN_MINIMUM_WORK_BUDGET})
|
|
255
|
+
local originals = redis.call('ZRANGEBYLEX', KEYS[1], lower, upper, 'LIMIT', 0, workLimit + 1)
|
|
235
256
|
local emitted = 0
|
|
236
257
|
local encodedBytes = 0
|
|
237
258
|
local examined = 0
|
|
238
|
-
local result = {'
|
|
259
|
+
local result = {'${REDIS_STORE_TOKEN.Page}', tostring(snapshotNow), ''}
|
|
239
260
|
while examined < #originals and examined < workLimit and emitted < limit do
|
|
240
261
|
local original = originals[examined + 1]
|
|
241
262
|
examined = examined + 1
|
|
242
|
-
|
|
243
|
-
|
|
244
|
-
|
|
245
|
-
|
|
246
|
-
|
|
247
|
-
|
|
248
|
-
|
|
249
|
-
|
|
250
|
-
|
|
251
|
-
|
|
252
|
-
|
|
253
|
-
|
|
254
|
-
break
|
|
255
|
-
end
|
|
256
|
-
table.insert(result, original)
|
|
257
|
-
table.insert(result, record[2])
|
|
258
|
-
table.insert(result, record[3])
|
|
259
|
-
table.insert(result, tostring(expiresAtMs))
|
|
260
|
-
encodedBytes = encodedBytes + itemBytes
|
|
261
|
-
emitted = emitted + 1
|
|
263
|
+
local rowKey = redis.call('HGET', KEYS[2], original)
|
|
264
|
+
if rowKey then
|
|
265
|
+
local members = redis.call('ZREVRANGEBYSCORE', rowKey, boundary, '-inf', 'LIMIT', 0, 1)
|
|
266
|
+
if #members > 0 then
|
|
267
|
+
local record = decodeMember(members[1])
|
|
268
|
+
local expiresAtMs = tonumber(record[4])
|
|
269
|
+
if record[1] == original and record[5] ~= true
|
|
270
|
+
and (expiresAtMs == 0 or expiresAtMs > snapshotNow) then
|
|
271
|
+
local itemBytes = string.len(original) + string.len(record[2]) + string.len(record[3]) + ${REDIS_ENCODED_ITEM_OVERHEAD_BYTES}
|
|
272
|
+
if emitted > 0 and encodedBytes + itemBytes > ${ZLINK_PROVIDER_MAX_ENCODED_PAGE_BYTES} then
|
|
273
|
+
examined = examined - 1
|
|
274
|
+
break
|
|
262
275
|
end
|
|
276
|
+
table.insert(result, original)
|
|
277
|
+
table.insert(result, record[2])
|
|
278
|
+
table.insert(result, record[3])
|
|
279
|
+
table.insert(result, tostring(expiresAtMs))
|
|
280
|
+
encodedBytes = encodedBytes + itemBytes
|
|
281
|
+
emitted = emitted + 1
|
|
263
282
|
end
|
|
264
283
|
end
|
|
265
284
|
end
|
|
@@ -287,14 +306,14 @@ local limit = tonumber(ARGV[2])
|
|
|
287
306
|
local snapshotId = ARGV[3]
|
|
288
307
|
local lastKey = ''
|
|
289
308
|
|
|
290
|
-
if redis.call('ZCARD', KEYS[6]) >=
|
|
291
|
-
return {'
|
|
309
|
+
if redis.call('ZCARD', KEYS[6]) >= ${REDIS_MAX_SCAN_SNAPSHOTS} then
|
|
310
|
+
return {'${REDIS_STORE_TOKEN.Capacity}'}
|
|
292
311
|
end
|
|
293
312
|
redis.call('DEL', KEYS[3])
|
|
294
313
|
local boundary = tonumber(redis.call('GET', KEYS[5]) or '0')
|
|
295
314
|
redis.call('HSET', KEYS[3], 'now', tostring(nowMs), 'boundary', tostring(boundary), 'prefix', prefix)
|
|
296
|
-
redis.call('PEXPIRE', KEYS[3],
|
|
297
|
-
redis.call('ZADD', KEYS[6], nowMs +
|
|
315
|
+
redis.call('PEXPIRE', KEYS[3], ${REDIS_SCAN_SNAPSHOT_RETENTION_MS})
|
|
316
|
+
redis.call('ZADD', KEYS[6], nowMs + ${REDIS_SCAN_SNAPSHOT_RETENTION_MS}, snapshotId)
|
|
298
317
|
redis.call('ZADD', KEYS[7], boundary, snapshotId)
|
|
299
318
|
` +
|
|
300
319
|
SCAN_PAGE_READ;
|
|
@@ -312,7 +331,7 @@ local snapshotId = ARGV[4]
|
|
|
312
331
|
if redis.call('EXISTS', KEYS[3]) == 0 then
|
|
313
332
|
redis.call('ZREM', KEYS[6], snapshotId)
|
|
314
333
|
redis.call('ZREM', KEYS[7], snapshotId)
|
|
315
|
-
return {'
|
|
334
|
+
return {'${REDIS_STORE_TOKEN.Expired}'}
|
|
316
335
|
end
|
|
317
336
|
` +
|
|
318
337
|
SCAN_PAGE_READ;
|
|
@@ -328,15 +347,15 @@ export const BLOB_PUT_SCRIPT =
|
|
|
328
347
|
local existing = redis.call('GET', KEYS[1])
|
|
329
348
|
if existing then
|
|
330
349
|
if existing ~= ARGV[1] then
|
|
331
|
-
return {'
|
|
350
|
+
return {'${REDIS_STORE_TOKEN.Conflict}', nowMs}
|
|
332
351
|
end
|
|
333
352
|
local ttl = redis.call('PTTL', KEYS[1])
|
|
334
353
|
local expiresAtMs = nowMs + math.max(ttl, 0)
|
|
335
|
-
return {'
|
|
354
|
+
return {'${REDIS_STORE_TOKEN.AlreadyStored}', nowMs, tostring(expiresAtMs)}
|
|
336
355
|
end
|
|
337
356
|
local retentionMs = tonumber(ARGV[2])
|
|
338
357
|
redis.call('SET', KEYS[1], ARGV[1], 'PX', retentionMs)
|
|
339
|
-
return {'
|
|
358
|
+
return {'${REDIS_STORE_TOKEN.Stored}', nowMs, tostring(nowMs + retentionMs)}
|
|
340
359
|
`;
|
|
341
360
|
|
|
342
361
|
export const BLOB_READ_SCRIPT =
|