@bypixeltv/kvstore 1.1.0 → 1.2.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +42 -0
- package/dist/cursor.d.ts +5 -0
- package/dist/cursor.js +15 -0
- package/dist/index.d.ts +25 -7
- package/dist/index.js +100 -22
- package/dist/lease.d.ts +25 -0
- package/dist/lease.js +142 -0
- package/dist/namespace-cache.d.ts +20 -0
- package/dist/namespace-cache.js +145 -0
- package/dist/retry.d.ts +2 -0
- package/dist/retry.js +13 -0
- package/dist/transaction.d.ts +47 -0
- package/dist/transaction.js +71 -0
- package/package.json +1 -1
package/README.md
CHANGED
|
@@ -28,3 +28,45 @@ Use `controller.abort()` to unsubscribe. Use `watch` and `cursorFromChange` for
|
|
|
28
28
|
and replay after a disconnect. Connect to the router for a sharded deployment.
|
|
29
29
|
See the [KVStore SDK documentation](https://github.com/byPixelTV/kvstore#sdks) for recovery,
|
|
30
30
|
wire types and release channels.
|
|
31
|
+
|
|
32
|
+
## Conditional transactions and locks
|
|
33
|
+
|
|
34
|
+
```ts
|
|
35
|
+
await kvstore.withKeyLock("players", "alice", async () => {
|
|
36
|
+
await kvstore.transaction(async tx => {
|
|
37
|
+
const player = await tx.getJson<{ coins: number }>("players", "alice");
|
|
38
|
+
tx.setJson("players", "alice", { coins: (player?.coins ?? 0) + 1 });
|
|
39
|
+
});
|
|
40
|
+
});
|
|
41
|
+
```
|
|
42
|
+
|
|
43
|
+
`transaction` runs the callback once. Commit checks every record read by the callback
|
|
44
|
+
and every lock held in its async scope. A failed check throws `TransactionConflictError`
|
|
45
|
+
and commits no writes. For explicit request IDs and retries, use `transact` with the same
|
|
46
|
+
ID, checks, and operations. The transaction write methods accept absolute expiry times in
|
|
47
|
+
milliseconds, like the point write methods.
|
|
48
|
+
|
|
49
|
+
Locks have a renewable TTL of at least five seconds. The callback receives an
|
|
50
|
+
`AbortSignal` that cancels when the lease is lost or the caller aborts. Use
|
|
51
|
+
`transaction` for writes that must validate lease ownership at commit. Point writes
|
|
52
|
+
remain unconditional. Allow read and write access to the `kvstore-leases` namespace for
|
|
53
|
+
clients that acquire locks. Lock keys are compatible with the Go and Kotlin SDKs.
|
|
54
|
+
|
|
55
|
+
## Local namespace cache
|
|
56
|
+
|
|
57
|
+
```ts
|
|
58
|
+
const players = await kvstore.loadNamespaceCache("players");
|
|
59
|
+
try {
|
|
60
|
+
console.log(players.get("alice")?.value.toString("utf8"));
|
|
61
|
+
console.log(players.snapshot().size);
|
|
62
|
+
} finally {
|
|
63
|
+
players.close();
|
|
64
|
+
await players.finished;
|
|
65
|
+
}
|
|
66
|
+
```
|
|
67
|
+
|
|
68
|
+
The cache loads a completed scan and follows changes from its checkpoint. On a transient
|
|
69
|
+
disconnect it replays from the last cursor. An invalid cursor invalidates the local
|
|
70
|
+
snapshot until a fresh scan completes. `get` and `snapshot` exclude expired records.
|
|
71
|
+
Use `ready` to distinguish an invalid snapshot from an empty namespace and `finished`
|
|
72
|
+
to observe terminal stream errors or wait for shutdown.
|
package/dist/cursor.d.ts
ADDED
|
@@ -0,0 +1,5 @@
|
|
|
1
|
+
import type { Change, Cursor, ScanRecord } from "./index.js";
|
|
2
|
+
export declare function cursorFromChange(change: Change): Cursor;
|
|
3
|
+
/** Use the checkpoint from a completed scan. Sharded scans checkpoint at the end. */
|
|
4
|
+
export declare function cursorFromScan(checkpoint: ScanRecord): Cursor;
|
|
5
|
+
export declare function needsRescan(error: unknown): boolean;
|
package/dist/cursor.js
ADDED
|
@@ -0,0 +1,15 @@
|
|
|
1
|
+
import * as grpc from "@grpc/grpc-js";
|
|
2
|
+
import { errorCode } from "./retry.js";
|
|
3
|
+
export function cursorFromChange(change) {
|
|
4
|
+
return change.cursor ? { vector: change.cursor } : { revision: change.revision };
|
|
5
|
+
}
|
|
6
|
+
/** Use the checkpoint from a completed scan. Sharded scans checkpoint at the end. */
|
|
7
|
+
export function cursorFromScan(checkpoint) {
|
|
8
|
+
if (!checkpoint.checkpoint)
|
|
9
|
+
throw new TypeError("Expected a scan checkpoint");
|
|
10
|
+
return checkpoint.cursor ? { vector: checkpoint.cursor } : { revision: checkpoint.snapshotRevision };
|
|
11
|
+
}
|
|
12
|
+
export function needsRescan(error) {
|
|
13
|
+
const code = errorCode(error);
|
|
14
|
+
return code === grpc.status.OUT_OF_RANGE || code === grpc.status.FAILED_PRECONDITION;
|
|
15
|
+
}
|
package/dist/index.d.ts
CHANGED
|
@@ -1,3 +1,12 @@
|
|
|
1
|
+
import { type LockOptions } from "./lease.js";
|
|
2
|
+
import { NamespaceCache } from "./namespace-cache.js";
|
|
3
|
+
import { type Transaction, type TransactionRequest } from "./transaction.js";
|
|
4
|
+
export { cursorFromChange, cursorFromScan, needsRescan } from "./cursor.js";
|
|
5
|
+
export { LeaseLostError, defaultLockTtlMs, leaseNamespace } from "./lease.js";
|
|
6
|
+
export type { LockOptions } from "./lease.js";
|
|
7
|
+
export { NamespaceCache } from "./namespace-cache.js";
|
|
8
|
+
export { TransactionConflictError } from "./transaction.js";
|
|
9
|
+
export type { Transaction, TransactionCheck, TransactionRequest } from "./transaction.js";
|
|
1
10
|
export interface ClientOptions {
|
|
2
11
|
apiKey?: string;
|
|
3
12
|
plaintext?: boolean;
|
|
@@ -70,23 +79,33 @@ export interface BatchResult extends DeleteResult {
|
|
|
70
79
|
grpcCode: number;
|
|
71
80
|
error: string;
|
|
72
81
|
}
|
|
73
|
-
export declare function cursorFromChange(change: Change): Cursor;
|
|
74
|
-
/** Use the checkpoint from a completed scan. Sharded scans checkpoint at the end. */
|
|
75
|
-
export declare function cursorFromScan(checkpoint: ScanRecord): Cursor;
|
|
76
|
-
export declare function needsRescan(error: unknown): boolean;
|
|
77
82
|
export declare class KVStoreClient {
|
|
78
83
|
private readonly rpc;
|
|
79
84
|
private readonly metadata;
|
|
80
85
|
private readonly timeoutMs;
|
|
86
|
+
private readonly lifetime;
|
|
87
|
+
private readonly leases;
|
|
81
88
|
constructor(target: string, { apiKey, plaintext, tls, timeoutMs }?: ClientOptions);
|
|
82
89
|
close(): void;
|
|
83
90
|
private unary;
|
|
84
|
-
get(namespace: string, key: string, consistency?: "strong" | "eventual"): Promise<GetResult>;
|
|
91
|
+
get(namespace: string, key: string, consistency?: "strong" | "eventual", signal?: AbortSignal): Promise<GetResult>;
|
|
85
92
|
set(namespace: string, key: string, value: Uint8Array, expiresAtUnixMs?: number): Promise<SetResult>;
|
|
86
93
|
delete(namespace: string, key: string): Promise<DeleteResult>;
|
|
87
94
|
/** Results are ordered. A transport failure can leave the outcome unknown. */
|
|
88
95
|
batchWrite(operations: BatchOperation[]): Promise<BatchResult[]>;
|
|
89
|
-
|
|
96
|
+
/** Retry an unknown outcome with the same request ID and exact input. */
|
|
97
|
+
transact(request: TransactionRequest, signal?: AbortSignal): Promise<boolean>;
|
|
98
|
+
/** Run the callback once, then atomically validate its read set and apply its writes. */
|
|
99
|
+
transaction<T>(callback: (transaction: Transaction) => T | Promise<T>, signal?: AbortSignal): Promise<T>;
|
|
100
|
+
/** Keep an exclusive, renewable lease for callback-scoped transactions. */
|
|
101
|
+
withKeyLock<T>(namespace: string, key: string, callback: (signal: AbortSignal) => T | Promise<T>, options?: LockOptions): Promise<T>;
|
|
102
|
+
/** Load a local snapshot and maintain it through changefeed replay. */
|
|
103
|
+
loadNamespaceCache(namespace: string, signal?: AbortSignal): Promise<NamespaceCache>;
|
|
104
|
+
private readForTransaction;
|
|
105
|
+
private transactWithRetry;
|
|
106
|
+
getJson<T>(namespace: string, key: string, options?: {
|
|
107
|
+
emptyAsMissing?: boolean;
|
|
108
|
+
}): Promise<T | undefined>;
|
|
90
109
|
setJson<T>(namespace: string, key: string, value: T, expiresAtUnixMs?: number): Promise<SetResult>;
|
|
91
110
|
head(signal?: AbortSignal): Promise<Head>;
|
|
92
111
|
scan(namespace: string, signal?: AbortSignal): AsyncGenerator<ScanRecord>;
|
|
@@ -95,4 +114,3 @@ export declare class KVStoreClient {
|
|
|
95
114
|
/** Iterate over committed changes. Use watch when you need cursor checkpoints. */
|
|
96
115
|
onChange(namespace: string, options?: WatchOptions): AsyncGenerator<Change>;
|
|
97
116
|
}
|
|
98
|
-
export {};
|
package/dist/index.js
CHANGED
|
@@ -1,6 +1,15 @@
|
|
|
1
1
|
import * as grpc from "@grpc/grpc-js";
|
|
2
2
|
import { loadSync } from "@grpc/proto-loader";
|
|
3
|
+
import { randomUUID } from "node:crypto";
|
|
3
4
|
import { fileURLToPath } from "node:url";
|
|
5
|
+
import { LeaseManager } from "./lease.js";
|
|
6
|
+
import { NamespaceCache } from "./namespace-cache.js";
|
|
7
|
+
import { errorCode, pause } from "./retry.js";
|
|
8
|
+
import { StagedTransaction, TransactionConflictError, } from "./transaction.js";
|
|
9
|
+
export { cursorFromChange, cursorFromScan, needsRescan } from "./cursor.js";
|
|
10
|
+
export { LeaseLostError, defaultLockTtlMs, leaseNamespace } from "./lease.js";
|
|
11
|
+
export { NamespaceCache } from "./namespace-cache.js";
|
|
12
|
+
export { TransactionConflictError } from "./transaction.js";
|
|
4
13
|
const definition = loadSync(fileURLToPath(new URL("./kvstore.proto", import.meta.url)), {
|
|
5
14
|
defaults: true,
|
|
6
15
|
longs: String,
|
|
@@ -12,6 +21,11 @@ function expiration(ms) {
|
|
|
12
21
|
}
|
|
13
22
|
return String(ms);
|
|
14
23
|
}
|
|
24
|
+
function encodeOperations(operations) {
|
|
25
|
+
return operations.map(op => op.delete
|
|
26
|
+
? { namespace: op.namespace, key: op.key, delete: true }
|
|
27
|
+
: { namespace: op.namespace, key: op.key, value: op.value, expiresAtUnixMs: expiration(op.expiresAtUnixMs ?? 0) });
|
|
28
|
+
}
|
|
15
29
|
async function* frames(open, signal) {
|
|
16
30
|
signal?.throwIfAborted();
|
|
17
31
|
const stream = open();
|
|
@@ -27,24 +41,12 @@ async function* frames(open, signal) {
|
|
|
27
41
|
stream.cancel();
|
|
28
42
|
}
|
|
29
43
|
}
|
|
30
|
-
export function cursorFromChange(change) {
|
|
31
|
-
return change.cursor ? { vector: change.cursor } : { revision: change.revision };
|
|
32
|
-
}
|
|
33
|
-
/** Use the checkpoint from a completed scan. Sharded scans checkpoint at the end. */
|
|
34
|
-
export function cursorFromScan(checkpoint) {
|
|
35
|
-
if (!checkpoint.checkpoint)
|
|
36
|
-
throw new TypeError("Expected a scan checkpoint");
|
|
37
|
-
return checkpoint.cursor ? { vector: checkpoint.cursor } : { revision: checkpoint.snapshotRevision };
|
|
38
|
-
}
|
|
39
|
-
export function needsRescan(error) {
|
|
40
|
-
if (typeof error !== "object" || error === null || !("code" in error))
|
|
41
|
-
return false;
|
|
42
|
-
return error.code === grpc.status.OUT_OF_RANGE || error.code === grpc.status.FAILED_PRECONDITION;
|
|
43
|
-
}
|
|
44
44
|
export class KVStoreClient {
|
|
45
45
|
rpc;
|
|
46
46
|
metadata = new grpc.Metadata();
|
|
47
47
|
timeoutMs;
|
|
48
|
+
lifetime = new AbortController();
|
|
49
|
+
leases;
|
|
48
50
|
constructor(target, { apiKey, plaintext = false, tls, timeoutMs = 5000 } = {}) {
|
|
49
51
|
if (plaintext && tls)
|
|
50
52
|
throw new TypeError("plaintext and tls are mutually exclusive");
|
|
@@ -58,8 +60,10 @@ export class KVStoreClient {
|
|
|
58
60
|
this.rpc = new service.kvstore.v1.KVStore(target, credentials);
|
|
59
61
|
if (apiKey)
|
|
60
62
|
this.metadata.set("authorization", `Bearer ${apiKey}`);
|
|
63
|
+
this.leases = new LeaseManager(this.lifetime.signal, (namespace, key, signal) => this.readForTransaction(namespace, key, signal), (checks, operations, signal) => this.transactWithRetry(checks, operations, signal));
|
|
61
64
|
}
|
|
62
65
|
close() {
|
|
66
|
+
this.lifetime.abort();
|
|
63
67
|
this.rpc.close();
|
|
64
68
|
}
|
|
65
69
|
unary(rpc, request, signal) {
|
|
@@ -76,8 +80,8 @@ export class KVStoreClient {
|
|
|
76
80
|
signal?.addEventListener("abort", abort, { once: true });
|
|
77
81
|
});
|
|
78
82
|
}
|
|
79
|
-
get(namespace, key, consistency = "strong") {
|
|
80
|
-
return this.unary(this.rpc.get, { namespace, key, consistency: consistency === "strong" ? 0 : 1 });
|
|
83
|
+
get(namespace, key, consistency = "strong", signal) {
|
|
84
|
+
return this.unary(this.rpc.get, { namespace, key, consistency: consistency === "strong" ? 0 : 1 }, signal);
|
|
81
85
|
}
|
|
82
86
|
set(namespace, key, value, expiresAtUnixMs = 0) {
|
|
83
87
|
return this.unary(this.rpc.set, { namespace, key, value, expiresAtUnixMs: expiration(expiresAtUnixMs) });
|
|
@@ -87,14 +91,88 @@ export class KVStoreClient {
|
|
|
87
91
|
}
|
|
88
92
|
/** Results are ordered. A transport failure can leave the outcome unknown. */
|
|
89
93
|
async batchWrite(operations) {
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
|
|
94
|
+
return (await this.unary(this.rpc.batchWrite, { operations: encodeOperations(operations) })).results;
|
|
95
|
+
}
|
|
96
|
+
/** Retry an unknown outcome with the same request ID and exact input. */
|
|
97
|
+
async transact(request, signal) {
|
|
98
|
+
const reply = await this.unary(this.rpc.transact, {
|
|
99
|
+
requestId: request.requestId,
|
|
100
|
+
checks: [...request.checks ?? []],
|
|
101
|
+
operations: encodeOperations(request.operations ?? []),
|
|
102
|
+
}, signal);
|
|
103
|
+
return reply.committed;
|
|
104
|
+
}
|
|
105
|
+
/** Run the callback once, then atomically validate its read set and apply its writes. */
|
|
106
|
+
async transaction(callback, signal) {
|
|
107
|
+
const active = this.leases.signal(signal);
|
|
108
|
+
active.throwIfAborted();
|
|
109
|
+
const transaction = new StagedTransaction(active, (namespace, key) => this.readForTransaction(namespace, key, active));
|
|
110
|
+
try {
|
|
111
|
+
const result = await callback(transaction);
|
|
112
|
+
const { checks, operations } = await transaction.prepare();
|
|
113
|
+
checks.unshift(...this.leases.checks);
|
|
114
|
+
if (checks.length + operations.length > 0 && !(await this.transactWithRetry(checks, operations, active))) {
|
|
115
|
+
throw new TransactionConflictError();
|
|
116
|
+
}
|
|
117
|
+
return result;
|
|
118
|
+
}
|
|
119
|
+
finally {
|
|
120
|
+
transaction.close();
|
|
121
|
+
}
|
|
122
|
+
}
|
|
123
|
+
/** Keep an exclusive, renewable lease for callback-scoped transactions. */
|
|
124
|
+
withKeyLock(namespace, key, callback, options = {}) {
|
|
125
|
+
return this.leases.withKeyLock(namespace, key, callback, options);
|
|
126
|
+
}
|
|
127
|
+
/** Load a local snapshot and maintain it through changefeed replay. */
|
|
128
|
+
loadNamespaceCache(namespace, signal) {
|
|
129
|
+
const active = signal ? AbortSignal.any([signal, this.lifetime.signal]) : this.lifetime.signal;
|
|
130
|
+
return NamespaceCache.open(this, namespace, active);
|
|
131
|
+
}
|
|
132
|
+
async readForTransaction(namespace, key, signal) {
|
|
133
|
+
const bounded = AbortSignal.any([signal, AbortSignal.timeout(this.timeoutMs)]);
|
|
134
|
+
for (;;) {
|
|
135
|
+
try {
|
|
136
|
+
return await this.get(namespace, key, "strong", bounded);
|
|
137
|
+
}
|
|
138
|
+
catch (error) {
|
|
139
|
+
bounded.throwIfAborted();
|
|
140
|
+
if (errorCode(error) !== grpc.status.UNAVAILABLE)
|
|
141
|
+
throw error;
|
|
142
|
+
}
|
|
143
|
+
await pause(25, bounded);
|
|
144
|
+
}
|
|
145
|
+
}
|
|
146
|
+
async transactWithRetry(checks, operations, signal) {
|
|
147
|
+
signal.throwIfAborted();
|
|
148
|
+
const request = { requestId: randomUUID(), checks, operations };
|
|
149
|
+
const bounded = AbortSignal.any([signal, AbortSignal.timeout(10_000)]);
|
|
150
|
+
try {
|
|
151
|
+
for (;;) {
|
|
152
|
+
try {
|
|
153
|
+
return await this.transact(request, bounded);
|
|
154
|
+
}
|
|
155
|
+
catch (error) {
|
|
156
|
+
bounded.throwIfAborted();
|
|
157
|
+
const code = errorCode(error);
|
|
158
|
+
if (code !== grpc.status.UNAVAILABLE && code !== grpc.status.DEADLINE_EXCEEDED && code !== grpc.status.ABORTED) {
|
|
159
|
+
throw error;
|
|
160
|
+
}
|
|
161
|
+
}
|
|
162
|
+
await pause(25, bounded);
|
|
163
|
+
}
|
|
164
|
+
}
|
|
165
|
+
catch (error) {
|
|
166
|
+
if (bounded.aborted)
|
|
167
|
+
throw new Error(`KVStore transaction ${request.requestId} outcome unknown`, { cause: error });
|
|
168
|
+
throw error;
|
|
169
|
+
}
|
|
94
170
|
}
|
|
95
|
-
async getJson(namespace, key) {
|
|
171
|
+
async getJson(namespace, key, options = {}) {
|
|
96
172
|
const result = await this.get(namespace, key);
|
|
97
|
-
|
|
173
|
+
if (!result.found || (options.emptyAsMissing && result.value.length === 0))
|
|
174
|
+
return undefined;
|
|
175
|
+
return JSON.parse(result.value.toString("utf8"));
|
|
98
176
|
}
|
|
99
177
|
async setJson(namespace, key, value, expiresAtUnixMs = 0) {
|
|
100
178
|
const json = JSON.stringify(value);
|
package/dist/lease.d.ts
ADDED
|
@@ -0,0 +1,25 @@
|
|
|
1
|
+
import type { BatchOperation, GetResult } from "./index.js";
|
|
2
|
+
import type { TransactionCheck } from "./transaction.js";
|
|
3
|
+
export declare const leaseNamespace = "kvstore-leases";
|
|
4
|
+
export declare const defaultLockTtlMs = 10000;
|
|
5
|
+
export declare class LeaseLostError extends Error {
|
|
6
|
+
constructor(options?: ErrorOptions);
|
|
7
|
+
}
|
|
8
|
+
export interface LockOptions {
|
|
9
|
+
ttlMs?: number;
|
|
10
|
+
signal?: AbortSignal;
|
|
11
|
+
}
|
|
12
|
+
export declare class LeaseManager {
|
|
13
|
+
private readonly lifetime;
|
|
14
|
+
private readonly read;
|
|
15
|
+
private readonly transact;
|
|
16
|
+
private readonly held;
|
|
17
|
+
constructor(lifetime: AbortSignal, read: (namespace: string, key: string, signal: AbortSignal) => Promise<GetResult>, transact: (checks: TransactionCheck[], writes: BatchOperation[], signal: AbortSignal) => Promise<boolean>);
|
|
18
|
+
get checks(): TransactionCheck[];
|
|
19
|
+
signal(signal?: AbortSignal): AbortSignal;
|
|
20
|
+
withKeyLock<T>(namespace: string, key: string, callback: (signal: AbortSignal) => T | Promise<T>, { signal, ttlMs }: LockOptions): Promise<T>;
|
|
21
|
+
private acquire;
|
|
22
|
+
private renew;
|
|
23
|
+
private writeIfOwned;
|
|
24
|
+
private write;
|
|
25
|
+
}
|
package/dist/lease.js
ADDED
|
@@ -0,0 +1,142 @@
|
|
|
1
|
+
import { AsyncLocalStorage } from "node:async_hooks";
|
|
2
|
+
import { randomUUID } from "node:crypto";
|
|
3
|
+
import { pause } from "./retry.js";
|
|
4
|
+
export const leaseNamespace = "kvstore-leases";
|
|
5
|
+
export const defaultLockTtlMs = 10_000;
|
|
6
|
+
const transactionHorizonMs = 3_000;
|
|
7
|
+
export class LeaseLostError extends Error {
|
|
8
|
+
constructor(options) {
|
|
9
|
+
super("KVStore distributed lock was lost", options);
|
|
10
|
+
this.name = "LeaseLostError";
|
|
11
|
+
}
|
|
12
|
+
}
|
|
13
|
+
function ownerCheck(lock) {
|
|
14
|
+
return { namespace: leaseNamespace, key: lock.key, value: lock.value };
|
|
15
|
+
}
|
|
16
|
+
export class LeaseManager {
|
|
17
|
+
lifetime;
|
|
18
|
+
read;
|
|
19
|
+
transact;
|
|
20
|
+
held = new AsyncLocalStorage();
|
|
21
|
+
constructor(lifetime, read, transact) {
|
|
22
|
+
this.lifetime = lifetime;
|
|
23
|
+
this.read = read;
|
|
24
|
+
this.transact = transact;
|
|
25
|
+
}
|
|
26
|
+
get checks() {
|
|
27
|
+
return this.held.getStore()?.locks.map(ownerCheck) ?? [];
|
|
28
|
+
}
|
|
29
|
+
signal(signal) {
|
|
30
|
+
const held = this.held.getStore();
|
|
31
|
+
const active = held?.signal ?? this.lifetime;
|
|
32
|
+
return signal ? AbortSignal.any([active, signal]) : active;
|
|
33
|
+
}
|
|
34
|
+
async withKeyLock(namespace, key, callback, { signal, ttlMs = defaultLockTtlMs }) {
|
|
35
|
+
if (!namespace || !key || !Number.isSafeInteger(ttlMs + Date.now()) || !Number.isSafeInteger(ttlMs) || ttlMs < 5000) {
|
|
36
|
+
throw new RangeError("a lock requires a namespace, a key and a TTL of at least 5000 ms");
|
|
37
|
+
}
|
|
38
|
+
const active = this.signal(signal);
|
|
39
|
+
active.throwIfAborted();
|
|
40
|
+
const locks = this.held.getStore()?.locks ?? [];
|
|
41
|
+
// Matches the Go and Kotlin wire format exactly.
|
|
42
|
+
const encoded = Buffer.from(namespace).toString("base64url") + ":" + Buffer.from(key).toString("base64url");
|
|
43
|
+
if (locks.some(lock => lock.key === encoded)) {
|
|
44
|
+
const result = await this.held.run({ signal: active, locks }, () => callback(active));
|
|
45
|
+
active.throwIfAborted();
|
|
46
|
+
return result;
|
|
47
|
+
}
|
|
48
|
+
const lock = { key: encoded, value: Buffer.from(randomUUID()) };
|
|
49
|
+
const acquisition = AbortSignal.any([active, AbortSignal.timeout(defaultLockTtlMs)]);
|
|
50
|
+
let expiresAt = await this.acquire(lock, ttlMs, acquisition);
|
|
51
|
+
const completion = new AbortController();
|
|
52
|
+
const loss = new AbortController();
|
|
53
|
+
const owned = AbortSignal.any([active, completion.signal, loss.signal]);
|
|
54
|
+
const renewal = this.renew(lock, ttlMs, expiresAt, owned, next => { expiresAt = next; }).catch(error => {
|
|
55
|
+
if (!owned.aborted)
|
|
56
|
+
loss.abort(error instanceof LeaseLostError ? error : new LeaseLostError({ cause: error }));
|
|
57
|
+
});
|
|
58
|
+
const errors = [];
|
|
59
|
+
let result;
|
|
60
|
+
try {
|
|
61
|
+
if (Date.now() >= expiresAt)
|
|
62
|
+
throw new LeaseLostError();
|
|
63
|
+
owned.throwIfAborted();
|
|
64
|
+
result = await this.held.run({ signal: owned, locks: [lock, ...locks] }, () => callback(owned));
|
|
65
|
+
owned.throwIfAborted();
|
|
66
|
+
if (Date.now() >= expiresAt)
|
|
67
|
+
throw new LeaseLostError();
|
|
68
|
+
}
|
|
69
|
+
catch (error) {
|
|
70
|
+
errors.push(error);
|
|
71
|
+
}
|
|
72
|
+
finally {
|
|
73
|
+
completion.abort();
|
|
74
|
+
await renewal;
|
|
75
|
+
try {
|
|
76
|
+
const release = AbortSignal.any([this.lifetime, AbortSignal.timeout(5000)]);
|
|
77
|
+
if (!(await this.writeIfOwned(lock, 0, release)) &&
|
|
78
|
+
!errors.some(error => error instanceof LeaseLostError)) {
|
|
79
|
+
errors.push(new LeaseLostError());
|
|
80
|
+
}
|
|
81
|
+
}
|
|
82
|
+
catch (error) {
|
|
83
|
+
errors.push(error);
|
|
84
|
+
}
|
|
85
|
+
}
|
|
86
|
+
if (errors.length > 1)
|
|
87
|
+
throw new AggregateError(errors, "KVStore distributed lock callback and release failed");
|
|
88
|
+
if (errors.length === 1)
|
|
89
|
+
throw errors[0];
|
|
90
|
+
return result;
|
|
91
|
+
}
|
|
92
|
+
async acquire(lock, ttlMs, signal) {
|
|
93
|
+
const absent = { namespace: leaseNamespace, key: lock.key, revision: "0" };
|
|
94
|
+
for (;;) {
|
|
95
|
+
const existing = await this.read(leaseNamespace, lock.key, signal);
|
|
96
|
+
if (!existing.found) {
|
|
97
|
+
const expiresAt = Date.now() + ttlMs;
|
|
98
|
+
if (await this.write(lock, absent, expiresAt, signal))
|
|
99
|
+
return expiresAt;
|
|
100
|
+
}
|
|
101
|
+
await pause(25, signal);
|
|
102
|
+
}
|
|
103
|
+
}
|
|
104
|
+
async renew(lock, ttlMs, expiresAt, signal, renewed) {
|
|
105
|
+
// A conditional write must observe a lease valid beyond the server's
|
|
106
|
+
// transaction horizon. Renew early enough to leave room for a retry.
|
|
107
|
+
const interval = Math.floor(Math.min(ttlMs / 3, (ttlMs - transactionHorizonMs) / 2));
|
|
108
|
+
while (!signal.aborted) {
|
|
109
|
+
// Cap the timer to Node's signed 32-bit timer range.
|
|
110
|
+
await pause(Math.min(0x7fffffff, Math.max(0, expiresAt - ttlMs + interval - Date.now())), signal);
|
|
111
|
+
const remaining = expiresAt - Date.now();
|
|
112
|
+
if (remaining <= 0)
|
|
113
|
+
throw new LeaseLostError();
|
|
114
|
+
const renewal = AbortSignal.any([signal, AbortSignal.timeout(Math.min(remaining, defaultLockTtlMs))]);
|
|
115
|
+
const next = Date.now() + ttlMs;
|
|
116
|
+
if (!(await this.writeIfOwned(lock, next, renewal)))
|
|
117
|
+
throw new LeaseLostError();
|
|
118
|
+
expiresAt = next;
|
|
119
|
+
renewed(next);
|
|
120
|
+
}
|
|
121
|
+
}
|
|
122
|
+
async writeIfOwned(lock, expiry, signal) {
|
|
123
|
+
const check = ownerCheck(lock);
|
|
124
|
+
for (;;) {
|
|
125
|
+
if (await this.write(lock, check, expiry, signal))
|
|
126
|
+
return true;
|
|
127
|
+
// Another transaction may temporarily reserve the lease key. Only a
|
|
128
|
+
// strong read of a different owner (or absence) confirms that it is lost.
|
|
129
|
+
const existing = await this.read(leaseNamespace, lock.key, signal);
|
|
130
|
+
if (!existing.found || !existing.value.equals(lock.value))
|
|
131
|
+
return false;
|
|
132
|
+
await pause(25, signal);
|
|
133
|
+
}
|
|
134
|
+
}
|
|
135
|
+
write(lock, check, expiry, signal) {
|
|
136
|
+
const key = { namespace: leaseNamespace, key: lock.key };
|
|
137
|
+
const operation = expiry === 0
|
|
138
|
+
? { ...key, delete: true }
|
|
139
|
+
: { ...key, value: lock.value, expiresAtUnixMs: expiry };
|
|
140
|
+
return this.transact([check], [operation], signal);
|
|
141
|
+
}
|
|
142
|
+
}
|
|
@@ -0,0 +1,20 @@
|
|
|
1
|
+
import type { GetResult, KVStoreClient } from "./index.js";
|
|
2
|
+
type Transport = Pick<KVStoreClient, "scan" | "watch">;
|
|
3
|
+
/** A completed namespace scan maintained by a resumable Watch. */
|
|
4
|
+
export declare class NamespaceCache {
|
|
5
|
+
private readonly stop;
|
|
6
|
+
private records;
|
|
7
|
+
private generation;
|
|
8
|
+
private loaded;
|
|
9
|
+
readonly finished: Promise<void>;
|
|
10
|
+
private constructor();
|
|
11
|
+
static open(transport: Transport, namespace: string, signal: AbortSignal): Promise<NamespaceCache>;
|
|
12
|
+
get version(): number;
|
|
13
|
+
get ready(): boolean;
|
|
14
|
+
get(key: string): GetResult | undefined;
|
|
15
|
+
snapshot(): ReadonlyMap<string, GetResult>;
|
|
16
|
+
close(): void;
|
|
17
|
+
private clear;
|
|
18
|
+
private follow;
|
|
19
|
+
}
|
|
20
|
+
export {};
|
|
@@ -0,0 +1,145 @@
|
|
|
1
|
+
import * as grpc from "@grpc/grpc-js";
|
|
2
|
+
import { cursorFromChange, cursorFromScan, needsRescan } from "./cursor.js";
|
|
3
|
+
import { errorCode, pause } from "./retry.js";
|
|
4
|
+
function expired(record, now) {
|
|
5
|
+
const expiry = Number(record.expiresAtUnixMs);
|
|
6
|
+
return expiry > 0 && expiry <= now;
|
|
7
|
+
}
|
|
8
|
+
function transient(error) {
|
|
9
|
+
const code = errorCode(error);
|
|
10
|
+
return code === grpc.status.UNAVAILABLE || code === grpc.status.DEADLINE_EXCEEDED ||
|
|
11
|
+
code === grpc.status.ABORTED;
|
|
12
|
+
}
|
|
13
|
+
async function scan(transport, namespace, signal) {
|
|
14
|
+
const records = new Map();
|
|
15
|
+
let cursor;
|
|
16
|
+
for await (const record of transport.scan(namespace, signal)) {
|
|
17
|
+
if (record.checkpoint) {
|
|
18
|
+
cursor = cursorFromScan(record);
|
|
19
|
+
continue;
|
|
20
|
+
}
|
|
21
|
+
records.set(record.key, {
|
|
22
|
+
found: true, value: record.value, revision: record.revision, expiresAtUnixMs: record.expiresAtUnixMs,
|
|
23
|
+
});
|
|
24
|
+
}
|
|
25
|
+
signal.throwIfAborted();
|
|
26
|
+
if (!cursor)
|
|
27
|
+
throw new Error("KVStore scan ended without a checkpoint");
|
|
28
|
+
return { records, cursor };
|
|
29
|
+
}
|
|
30
|
+
/** A completed namespace scan maintained by a resumable Watch. */
|
|
31
|
+
export class NamespaceCache {
|
|
32
|
+
stop = new AbortController();
|
|
33
|
+
records;
|
|
34
|
+
generation = 1;
|
|
35
|
+
loaded = true;
|
|
36
|
+
finished;
|
|
37
|
+
constructor(transport, namespace, snapshot, parent) {
|
|
38
|
+
this.records = snapshot.records;
|
|
39
|
+
const signal = AbortSignal.any([parent, this.stop.signal]);
|
|
40
|
+
this.finished = this.follow(transport, namespace, snapshot.cursor, signal).catch(error => {
|
|
41
|
+
if (!signal.aborted)
|
|
42
|
+
throw error;
|
|
43
|
+
}).finally(() => this.clear());
|
|
44
|
+
// Consumers may use the cache synchronously and choose to inspect finished later.
|
|
45
|
+
void this.finished.catch(() => { });
|
|
46
|
+
}
|
|
47
|
+
static async open(transport, namespace, signal) {
|
|
48
|
+
const initial = AbortSignal.any([signal, AbortSignal.timeout(10_000)]);
|
|
49
|
+
for (;;) {
|
|
50
|
+
try {
|
|
51
|
+
return new NamespaceCache(transport, namespace, await scan(transport, namespace, initial), signal);
|
|
52
|
+
}
|
|
53
|
+
catch (error) {
|
|
54
|
+
initial.throwIfAborted();
|
|
55
|
+
if (!transient(error))
|
|
56
|
+
throw error;
|
|
57
|
+
await pause(25, initial);
|
|
58
|
+
}
|
|
59
|
+
}
|
|
60
|
+
}
|
|
61
|
+
get version() {
|
|
62
|
+
return this.generation;
|
|
63
|
+
}
|
|
64
|
+
get ready() {
|
|
65
|
+
return this.loaded;
|
|
66
|
+
}
|
|
67
|
+
get(key) {
|
|
68
|
+
const record = this.records.get(key);
|
|
69
|
+
if (!record)
|
|
70
|
+
return undefined;
|
|
71
|
+
if (expired(record, Date.now())) {
|
|
72
|
+
this.records.delete(key);
|
|
73
|
+
this.generation++;
|
|
74
|
+
return undefined;
|
|
75
|
+
}
|
|
76
|
+
return { ...record, value: Buffer.from(record.value) };
|
|
77
|
+
}
|
|
78
|
+
snapshot() {
|
|
79
|
+
const result = new Map();
|
|
80
|
+
const now = Date.now();
|
|
81
|
+
for (const [key, record] of this.records) {
|
|
82
|
+
if (expired(record, now)) {
|
|
83
|
+
this.records.delete(key);
|
|
84
|
+
this.generation++;
|
|
85
|
+
}
|
|
86
|
+
else {
|
|
87
|
+
result.set(key, { ...record, value: Buffer.from(record.value) });
|
|
88
|
+
}
|
|
89
|
+
}
|
|
90
|
+
return result;
|
|
91
|
+
}
|
|
92
|
+
close() {
|
|
93
|
+
this.stop.abort();
|
|
94
|
+
this.clear();
|
|
95
|
+
}
|
|
96
|
+
clear() {
|
|
97
|
+
if (this.loaded) {
|
|
98
|
+
this.records.clear();
|
|
99
|
+
this.loaded = false;
|
|
100
|
+
this.generation++;
|
|
101
|
+
}
|
|
102
|
+
}
|
|
103
|
+
async follow(transport, namespace, after, signal) {
|
|
104
|
+
let rescan = false;
|
|
105
|
+
while (!signal.aborted) {
|
|
106
|
+
try {
|
|
107
|
+
if (rescan) {
|
|
108
|
+
const next = await scan(transport, namespace, signal);
|
|
109
|
+
this.records = next.records;
|
|
110
|
+
this.loaded = true;
|
|
111
|
+
this.generation++;
|
|
112
|
+
after = next.cursor;
|
|
113
|
+
rescan = false;
|
|
114
|
+
}
|
|
115
|
+
for await (const change of transport.watch(namespace, { after, signal })) {
|
|
116
|
+
signal.throwIfAborted();
|
|
117
|
+
if (!change.checkpoint) {
|
|
118
|
+
if (change.deleted) {
|
|
119
|
+
if (this.records.delete(change.key))
|
|
120
|
+
this.generation++;
|
|
121
|
+
}
|
|
122
|
+
else if (this.records.get(change.key)?.revision !== change.revision) {
|
|
123
|
+
this.records.set(change.key, {
|
|
124
|
+
found: true, value: change.value, revision: change.revision, expiresAtUnixMs: change.expiresAtUnixMs,
|
|
125
|
+
});
|
|
126
|
+
this.generation++;
|
|
127
|
+
}
|
|
128
|
+
}
|
|
129
|
+
after = cursorFromChange(change);
|
|
130
|
+
}
|
|
131
|
+
}
|
|
132
|
+
catch (error) {
|
|
133
|
+
signal.throwIfAborted();
|
|
134
|
+
if (needsRescan(error)) {
|
|
135
|
+
this.clear();
|
|
136
|
+
rescan = true;
|
|
137
|
+
}
|
|
138
|
+
else if (!transient(error)) {
|
|
139
|
+
throw error;
|
|
140
|
+
}
|
|
141
|
+
}
|
|
142
|
+
await pause(25, signal);
|
|
143
|
+
}
|
|
144
|
+
}
|
|
145
|
+
}
|
package/dist/retry.d.ts
ADDED
package/dist/retry.js
ADDED
|
@@ -0,0 +1,13 @@
|
|
|
1
|
+
import { setTimeout as sleep } from "node:timers/promises";
|
|
2
|
+
export function errorCode(error) {
|
|
3
|
+
return typeof error === "object" && error !== null && "code" in error ? error.code : undefined;
|
|
4
|
+
}
|
|
5
|
+
export async function pause(ms, signal) {
|
|
6
|
+
try {
|
|
7
|
+
await sleep(ms, undefined, { signal });
|
|
8
|
+
}
|
|
9
|
+
catch (error) {
|
|
10
|
+
signal.throwIfAborted();
|
|
11
|
+
throw error;
|
|
12
|
+
}
|
|
13
|
+
}
|
|
@@ -0,0 +1,47 @@
|
|
|
1
|
+
import type { BatchOperation, GetResult } from "./index.js";
|
|
2
|
+
export type TransactionCheck = {
|
|
3
|
+
namespace: string;
|
|
4
|
+
key: string;
|
|
5
|
+
} & ({
|
|
6
|
+
revision: string;
|
|
7
|
+
value?: never;
|
|
8
|
+
} | {
|
|
9
|
+
value: Uint8Array;
|
|
10
|
+
revision?: never;
|
|
11
|
+
});
|
|
12
|
+
export interface TransactionRequest {
|
|
13
|
+
requestId: string;
|
|
14
|
+
checks?: readonly TransactionCheck[];
|
|
15
|
+
operations?: readonly BatchOperation[];
|
|
16
|
+
}
|
|
17
|
+
export declare class TransactionConflictError extends Error {
|
|
18
|
+
constructor();
|
|
19
|
+
}
|
|
20
|
+
export interface Transaction {
|
|
21
|
+
readonly signal: AbortSignal;
|
|
22
|
+
get(namespace: string, key: string): Promise<Buffer | undefined>;
|
|
23
|
+
getJson<T>(namespace: string, key: string): Promise<T | undefined>;
|
|
24
|
+
set(namespace: string, key: string, value: Uint8Array, expiresAtUnixMs?: number): void;
|
|
25
|
+
setJson<T>(namespace: string, key: string, value: T, expiresAtUnixMs?: number): void;
|
|
26
|
+
delete(namespace: string, key: string): void;
|
|
27
|
+
}
|
|
28
|
+
export declare class StagedTransaction implements Transaction {
|
|
29
|
+
readonly signal: AbortSignal;
|
|
30
|
+
private readonly read;
|
|
31
|
+
private readonly reads;
|
|
32
|
+
private readonly readsForCommit;
|
|
33
|
+
private readonly writes;
|
|
34
|
+
private open;
|
|
35
|
+
constructor(signal: AbortSignal, read: (namespace: string, key: string) => Promise<GetResult>);
|
|
36
|
+
get(namespace: string, key: string): Promise<Buffer | undefined>;
|
|
37
|
+
getJson<T>(namespace: string, key: string): Promise<T | undefined>;
|
|
38
|
+
set(namespace: string, key: string, value: Uint8Array, expiresAtUnixMs?: number): void;
|
|
39
|
+
setJson<T>(namespace: string, key: string, value: T, expiresAtUnixMs?: number): void;
|
|
40
|
+
delete(namespace: string, key: string): void;
|
|
41
|
+
prepare(): Promise<{
|
|
42
|
+
checks: TransactionCheck[];
|
|
43
|
+
operations: BatchOperation[];
|
|
44
|
+
}>;
|
|
45
|
+
close(): void;
|
|
46
|
+
private requireOpen;
|
|
47
|
+
}
|
|
@@ -0,0 +1,71 @@
|
|
|
1
|
+
export class TransactionConflictError extends Error {
|
|
2
|
+
constructor() {
|
|
3
|
+
super("KVStore transaction precondition failed");
|
|
4
|
+
this.name = "TransactionConflictError";
|
|
5
|
+
}
|
|
6
|
+
}
|
|
7
|
+
export class StagedTransaction {
|
|
8
|
+
signal;
|
|
9
|
+
read;
|
|
10
|
+
reads = new Map();
|
|
11
|
+
readsForCommit = [];
|
|
12
|
+
writes = new Map();
|
|
13
|
+
open = true;
|
|
14
|
+
constructor(signal, read) {
|
|
15
|
+
this.signal = signal;
|
|
16
|
+
this.read = read;
|
|
17
|
+
}
|
|
18
|
+
async get(namespace, key) {
|
|
19
|
+
this.requireOpen();
|
|
20
|
+
const ref = JSON.stringify([namespace, key]);
|
|
21
|
+
const write = this.writes.get(ref);
|
|
22
|
+
if (write)
|
|
23
|
+
return write.delete ? undefined : Buffer.from(write.value);
|
|
24
|
+
let read = this.reads.get(ref);
|
|
25
|
+
if (!read) {
|
|
26
|
+
read = this.read(namespace, key);
|
|
27
|
+
this.reads.set(ref, read);
|
|
28
|
+
const check = read.then(record => ({
|
|
29
|
+
namespace, key, revision: record.found ? record.revision : "0",
|
|
30
|
+
}));
|
|
31
|
+
// A callback can catch a failed read, or fail before awaiting it.
|
|
32
|
+
// The same error is observed before commit without an unhandled rejection.
|
|
33
|
+
void check.catch(() => { });
|
|
34
|
+
this.readsForCommit.push(check);
|
|
35
|
+
}
|
|
36
|
+
const record = await read;
|
|
37
|
+
return record.found ? Buffer.from(record.value) : undefined;
|
|
38
|
+
}
|
|
39
|
+
async getJson(namespace, key) {
|
|
40
|
+
const value = await this.get(namespace, key);
|
|
41
|
+
return value === undefined ? undefined : JSON.parse(value.toString("utf8"));
|
|
42
|
+
}
|
|
43
|
+
set(namespace, key, value, expiresAtUnixMs = 0) {
|
|
44
|
+
this.requireOpen();
|
|
45
|
+
this.writes.set(JSON.stringify([namespace, key]), { namespace, key, value: Buffer.from(value), expiresAtUnixMs });
|
|
46
|
+
}
|
|
47
|
+
setJson(namespace, key, value, expiresAtUnixMs = 0) {
|
|
48
|
+
const json = JSON.stringify(value);
|
|
49
|
+
if (json === undefined)
|
|
50
|
+
throw new TypeError("value has no JSON representation");
|
|
51
|
+
this.set(namespace, key, Buffer.from(json), expiresAtUnixMs);
|
|
52
|
+
}
|
|
53
|
+
delete(namespace, key) {
|
|
54
|
+
this.requireOpen();
|
|
55
|
+
this.writes.set(JSON.stringify([namespace, key]), { namespace, key, delete: true });
|
|
56
|
+
}
|
|
57
|
+
async prepare() {
|
|
58
|
+
this.close();
|
|
59
|
+
const checks = await Promise.all(this.readsForCommit);
|
|
60
|
+
this.signal.throwIfAborted();
|
|
61
|
+
return { checks, operations: [...this.writes.values()] };
|
|
62
|
+
}
|
|
63
|
+
close() {
|
|
64
|
+
this.open = false;
|
|
65
|
+
}
|
|
66
|
+
requireOpen() {
|
|
67
|
+
this.signal.throwIfAborted();
|
|
68
|
+
if (!this.open)
|
|
69
|
+
throw new Error("KVStore transaction callback has ended");
|
|
70
|
+
}
|
|
71
|
+
}
|