@midsummer-ai/sdk 0.0.0-stage → 0.1.0-next.419
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +177 -0
- package/README.md +4 -2
- package/agents/contract.d.mts +9 -0
- package/agents/contract.mjs +221 -0
- package/agents/index.d.mts +128 -0
- package/agents/index.mjs +203 -0
- package/agents/stand-in.d.mts +35 -0
- package/agents/stand-in.mjs +149 -0
- package/ai/contract.d.mts +10 -0
- package/ai/contract.mjs +77 -0
- package/ai/index.d.mts +94 -0
- package/ai/index.mjs +193 -0
- package/ai/stand-in.d.mts +7 -0
- package/ai/stand-in.mjs +41 -0
- package/build-SQ44jwSa.d.mts +41 -0
- package/client.d.mts +45 -0
- package/client.mjs +106 -0
- package/connection-CDk7H1FL.mjs +1871 -0
- package/contract.d.mts +10 -0
- package/contract.mjs +469 -0
- package/database/build.d.mts +2 -0
- package/database/build.mjs +579 -0
- package/database/client.d.mts +60 -0
- package/database/client.mjs +126 -0
- package/database/contract.d.mts +11 -0
- package/database/contract.mjs +164 -0
- package/database/errors.d.mts +23 -0
- package/database/errors.mjs +13 -0
- package/database/server.d.mts +3 -0
- package/database/server.mjs +320 -0
- package/database/stand-in.d.mts +96 -0
- package/database/stand-in.mjs +422 -0
- package/email/client.d.mts +34 -0
- package/email/client.mjs +131 -0
- package/email/contract.d.mts +11 -0
- package/email/contract.mjs +99 -0
- package/email/index.d.mts +74 -0
- package/email/index.mjs +59 -0
- package/email/smtp.d.mts +22 -0
- package/email/smtp.mjs +296 -0
- package/email/stand-in.d.mts +13 -0
- package/email/stand-in.mjs +51 -0
- package/index-DdoK0y7D.d.mts +192 -0
- package/index-sgdyoA2P.d.mts +212 -0
- package/index.d.mts +2 -0
- package/index.mjs +137 -0
- package/names-DfCBKupJ.mjs +115 -0
- package/package.json +152 -4
- package/predict-Drfm_8pO.mjs +179 -0
- package/queues/contract.d.mts +13 -0
- package/queues/contract.mjs +278 -0
- package/queues/index.d.mts +184 -0
- package/queues/index.mjs +126 -0
- package/queues/stand-in.d.mts +7 -0
- package/queues/stand-in.mjs +110 -0
- package/queues/wire.d.mts +9 -0
- package/queues/wire.mjs +199 -0
- package/s3.d.mts +43 -0
- package/s3.mjs +1003 -0
- package/schema-BF-Y1EVu.mjs +142 -0
- package/schema-DS8x1Dn8.mjs +115 -0
- package/schema-f7E9uzUn.d.mts +74 -0
- package/server-C7O7CICU.d.mts +182 -0
- package/socket.d.mts +67 -0
- package/socket.mjs +263 -0
- package/stand-in.d.mts +7 -0
- package/stand-in.mjs +185 -0
- package/store-BzI9at5x.mjs +237 -0
- package/tabs-D-P8_ci_.mjs +457 -0
- package/vector/contract.d.mts +10 -0
- package/vector/contract.mjs +635 -0
- package/vector/index.d.mts +2 -0
- package/vector/index.mjs +108 -0
- package/vector/stand-in.d.mts +11 -0
- package/vector/stand-in.mjs +169 -0
- package/vector/wire.d.mts +25 -0
- package/vector/wire.mjs +175 -0
- package/wire.d.mts +54 -0
- package/wire.mjs +731 -0
|
@@ -0,0 +1,184 @@
|
|
|
1
|
+
//#region src/queues/index.d.ts
|
|
2
|
+
/**
|
|
3
|
+
* Queues: a Team's Queues of Messages, which the SDK reaches by the name an Environment binds each one
|
|
4
|
+
* under (`midsummer env binding set`).
|
|
5
|
+
*
|
|
6
|
+
* import { queues } from "@midsummer-ai/sdk/queues"
|
|
7
|
+
* import { standIn } from "@midsummer-ai/sdk/queues/stand-in"
|
|
8
|
+
*
|
|
9
|
+
* const jobs = queues(standIn({ clock: Date.now }))
|
|
10
|
+
* const emails = jobs.queue("emails")
|
|
11
|
+
* await emails.send({ to: "ada@example.com" }, { key: `welcome:${userId}` })
|
|
12
|
+
* for (const delivery of await emails.receive({ limit: 10 })) {
|
|
13
|
+
* try { await mail(delivery.body); await emails.ack(delivery.receipt) }
|
|
14
|
+
* catch { await emails.retry(delivery.receipt) }
|
|
15
|
+
* }
|
|
16
|
+
* const { depth, oldestSentAt } = await emails.stats()
|
|
17
|
+
*
|
|
18
|
+
* The Stand-in keeps Messages in memory, for tests and `midsummer dev`. On Midsummer, a Consumer
|
|
19
|
+
* delivers each Message to a path of your Site (`midsummer queue consumer set`), and `remote` from
|
|
20
|
+
* `@midsummer-ai/sdk/queues/wire` reaches the real Queues over a transport. Each is held to the cases
|
|
21
|
+
* in `@midsummer-ai/sdk/queues/contract`; a behaviour that is not a case there is not promised.
|
|
22
|
+
*
|
|
23
|
+
* Delivery is at least once. A Message received is hidden from other receivers for its visibility
|
|
24
|
+
* window; acked, it is gone; retried, or left until the window ends, it is delivered again, a later
|
|
25
|
+
* time each attempt (10 seconds after the first, doubling, at most 10 minutes). Once it has been
|
|
26
|
+
* delivered `maxAttempts` times (5 unless the receiver says), a retry or an ended window moves it
|
|
27
|
+
* to the Queue's dead-letter Queue, `queue.deadLetter`, where it waits to be read and never moves on.
|
|
28
|
+
* A receipt names one delivery: acking or retrying one whose window has ended does nothing.
|
|
29
|
+
*
|
|
30
|
+
* A send with a `key` is made once: sending the same key to the same Queue again, within a day,
|
|
31
|
+
* returns the first Message's id and sends nothing. Schedules send with one, so a Schedule's minute
|
|
32
|
+
* sends once however often it is reached.
|
|
33
|
+
*
|
|
34
|
+
* Names and bodies are checked here, before anything leaves the caller. Every failure is a
|
|
35
|
+
* `QueueError`, whose `code` is the one the wire carries.
|
|
36
|
+
*/
|
|
37
|
+
/** Why a Queues call failed: as Storage's codes say (`invalid`, `forbidden`, `quota`, `unavailable` …). */
|
|
38
|
+
type QueueErrorCode = "invalid" | "forbidden" | "quota" | "unavailable";
|
|
39
|
+
declare const queueErrorCodes: ReadonlyArray<QueueErrorCode>;
|
|
40
|
+
declare class QueueError extends Error {
|
|
41
|
+
readonly code: QueueErrorCode;
|
|
42
|
+
constructor(code: QueueErrorCode, why: string);
|
|
43
|
+
}
|
|
44
|
+
/** Any value JSON can carry. */
|
|
45
|
+
type Json = string | number | boolean | null | ReadonlyArray<Json> | {
|
|
46
|
+
readonly [key: string]: Json;
|
|
47
|
+
};
|
|
48
|
+
/** One delivery of a Message: its body as sent, and the receipt that acks or retries it. */
|
|
49
|
+
interface Delivery<Body = Json> {
|
|
50
|
+
readonly id: string;
|
|
51
|
+
readonly body: Body;
|
|
52
|
+
/** When it was sent, in milliseconds since the epoch. */
|
|
53
|
+
readonly sentAt: number;
|
|
54
|
+
/** Which delivery this is: 1 the first time it is received. */
|
|
55
|
+
readonly attempt: number;
|
|
56
|
+
/** Names this delivery; it acks or retries it until `visibleUntil`, and does nothing after. */
|
|
57
|
+
readonly receipt: string;
|
|
58
|
+
/** When the visibility window ends and the Message may be delivered again. */
|
|
59
|
+
readonly visibleUntil: number;
|
|
60
|
+
}
|
|
61
|
+
interface SendOptions {
|
|
62
|
+
/** Seconds before it can be received: 0 to 43,200 (12 hours), 0 by default. */
|
|
63
|
+
readonly delaySeconds?: number;
|
|
64
|
+
/** Makes the send once: 1 to 128 printable characters. */
|
|
65
|
+
readonly key?: string;
|
|
66
|
+
}
|
|
67
|
+
interface ReceiveOptions {
|
|
68
|
+
/** At most this many: 1 to 100, 10 by default. */
|
|
69
|
+
readonly limit?: number;
|
|
70
|
+
/** Seconds each delivery is hidden from other receivers: 1 to 43,200, 30 by default. */
|
|
71
|
+
readonly visibilitySeconds?: number;
|
|
72
|
+
/** Deliveries after which a retry moves the Message to the dead-letter Queue: 1 to 100, 5 by default. */
|
|
73
|
+
readonly maxAttempts?: number;
|
|
74
|
+
}
|
|
75
|
+
/**
|
|
76
|
+
* What a Queue holds now. Each count stops at 1,000, with `more` set when there are more: reading
|
|
77
|
+
* stats never reads a whole Queue.
|
|
78
|
+
*/
|
|
79
|
+
interface QueueStats {
|
|
80
|
+
/** Every Message not yet acked: ready, in flight and delayed. */
|
|
81
|
+
readonly depth: number;
|
|
82
|
+
/** Messages that can be received now. */
|
|
83
|
+
readonly ready: number;
|
|
84
|
+
/** Messages delivered whose visibility window has not ended. */
|
|
85
|
+
readonly inFlight: number;
|
|
86
|
+
/** Messages sent with a delay, or retried, that cannot be received yet. */
|
|
87
|
+
readonly delayed: number;
|
|
88
|
+
/** When the oldest Message not yet acked was sent; absent when there is none. */
|
|
89
|
+
readonly oldestSentAt?: number;
|
|
90
|
+
/** Messages in the dead-letter Queue. */
|
|
91
|
+
readonly dead: number;
|
|
92
|
+
readonly more?: true;
|
|
93
|
+
}
|
|
94
|
+
/** The Messages of one Queue. */
|
|
95
|
+
interface Queue {
|
|
96
|
+
readonly name: string;
|
|
97
|
+
/** Sends one Message. Its body is any JSON value, at most 128 KiB as JSON. */
|
|
98
|
+
send(body: Json, options?: SendOptions): Promise<{
|
|
99
|
+
readonly id: string;
|
|
100
|
+
}>;
|
|
101
|
+
/** Sends up to 100 Messages together, received in the order given; ids in the same order. */
|
|
102
|
+
sendBatch(messages: ReadonlyArray<{
|
|
103
|
+
readonly body: Json;
|
|
104
|
+
} & SendOptions>): Promise<{
|
|
105
|
+
readonly ids: ReadonlyArray<string>;
|
|
106
|
+
}>;
|
|
107
|
+
/**
|
|
108
|
+
* Up to `limit` Messages that can be received now, oldest first. An engine looks at a bounded
|
|
109
|
+
* number of Messages a call, so it may answer fewer, or none, while it moves exhausted ones to the
|
|
110
|
+
* dead-letter Queue: receive again.
|
|
111
|
+
*/
|
|
112
|
+
receive<Body = Json>(options?: ReceiveOptions): Promise<ReadonlyArray<Delivery<Body>>>;
|
|
113
|
+
/** Ends a Message: it is not delivered again. */
|
|
114
|
+
ack(receipt: string): Promise<void>;
|
|
115
|
+
/**
|
|
116
|
+
* Hands a Message back to be delivered again after `delaySeconds`, or after the backoff when not
|
|
117
|
+
* given: 10 seconds after the first delivery, doubling, at most 600. Past its `maxAttempts` it
|
|
118
|
+
* moves to the dead-letter Queue instead.
|
|
119
|
+
*/
|
|
120
|
+
retry(receipt: string, options?: {
|
|
121
|
+
readonly delaySeconds?: number;
|
|
122
|
+
}): Promise<void>;
|
|
123
|
+
stats(): Promise<QueueStats>;
|
|
124
|
+
/** Where this Queue's Messages go once they have been delivered `maxAttempts` times. Its own Messages never move on. */
|
|
125
|
+
readonly deadLetter: Queue;
|
|
126
|
+
}
|
|
127
|
+
interface Queues {
|
|
128
|
+
/** A Queue by name: 1 to 63 lowercase letters, digits, hyphens and underscores, starting with a letter or digit. */
|
|
129
|
+
queue(name: string): Queue;
|
|
130
|
+
}
|
|
131
|
+
/** A Message as an engine is given it: its body as JSON text. */
|
|
132
|
+
interface EngineMessage {
|
|
133
|
+
readonly body: string;
|
|
134
|
+
readonly delayMs: number;
|
|
135
|
+
readonly key?: string;
|
|
136
|
+
}
|
|
137
|
+
/** A delivery as an engine returns it: its body as JSON text. */
|
|
138
|
+
interface EngineDelivery extends Omit<Delivery, "body"> {
|
|
139
|
+
readonly body: string;
|
|
140
|
+
}
|
|
141
|
+
/**
|
|
142
|
+
* Where Messages live: the Seam under `queues()`. The Stand-in and the real Capability are its
|
|
143
|
+
* Adapters. Calls arrive checked: a Queue's name is valid (with `.dead` after it for a dead-letter
|
|
144
|
+
* Queue), bodies are JSON text, times are milliseconds and defaults are filled in.
|
|
145
|
+
*/
|
|
146
|
+
interface QueueEngine {
|
|
147
|
+
/** Sends the Messages in order and returns their ids, a keyed one's first id when it was sent before. */
|
|
148
|
+
send(queue: string, messages: ReadonlyArray<EngineMessage>): Promise<ReadonlyArray<string>>;
|
|
149
|
+
receive(queue: string, options: {
|
|
150
|
+
readonly limit: number;
|
|
151
|
+
readonly visibilityMs: number;
|
|
152
|
+
readonly maxAttempts: number;
|
|
153
|
+
}): Promise<ReadonlyArray<EngineDelivery>>;
|
|
154
|
+
ack(queue: string, receipt: string): Promise<void>;
|
|
155
|
+
/** With no `delayMs`, the backoff after the delivery's attempt (`backoffMs`). */
|
|
156
|
+
retry(queue: string, receipt: string, delayMs?: number): Promise<void>;
|
|
157
|
+
stats(queue: string): Promise<QueueStats>;
|
|
158
|
+
}
|
|
159
|
+
/** The suffix that marks a Queue's dead-letter Queue; no Queue's own name has a dot. */
|
|
160
|
+
declare const DEAD = ".dead";
|
|
161
|
+
/** The delay before a Message's next delivery after its `attempt`-th: 10 seconds, doubling, at most 10 minutes. */
|
|
162
|
+
declare function backoffMs(attempt: number): number;
|
|
163
|
+
/** Stats' counts stop at this many. */
|
|
164
|
+
declare const STATS_CAP = 1000;
|
|
165
|
+
declare function invalid(why: string): never;
|
|
166
|
+
declare function checkQueueName(name: string): void;
|
|
167
|
+
/** A name as an engine is given it: a Queue's, or its dead-letter Queue's. */
|
|
168
|
+
declare function checkEngineQueue(queue: string): void;
|
|
169
|
+
declare function checkKey(key: string): void;
|
|
170
|
+
/** Bodies are at most this many bytes of JSON. */
|
|
171
|
+
declare const MAX_BODY: number;
|
|
172
|
+
/** The JSON text of `body`, which must be a JSON value of at most 128 KiB. */
|
|
173
|
+
declare function bodyText(body: unknown): string;
|
|
174
|
+
declare function checkReceipt(receipt: string): void;
|
|
175
|
+
declare function checkDelaySeconds(seconds: number): void;
|
|
176
|
+
declare function checkReceive(options: {
|
|
177
|
+
readonly limit: number;
|
|
178
|
+
readonly visibilityMs: number;
|
|
179
|
+
readonly maxAttempts: number;
|
|
180
|
+
}): void;
|
|
181
|
+
/** Queues over `engine`. */
|
|
182
|
+
declare function queues(engine: QueueEngine): Queues;
|
|
183
|
+
//#endregion
|
|
184
|
+
export { DEAD, Delivery, EngineDelivery, EngineMessage, Json, MAX_BODY, Queue, QueueEngine, QueueError, QueueErrorCode, QueueStats, Queues, ReceiveOptions, STATS_CAP, SendOptions, backoffMs, bodyText, checkDelaySeconds, checkEngineQueue, checkKey, checkQueueName, checkReceipt, checkReceive, invalid, queueErrorCodes, queues };
|
package/queues/index.mjs
ADDED
|
@@ -0,0 +1,126 @@
|
|
|
1
|
+
//#region src/queues/index.ts
|
|
2
|
+
const queueErrorCodes = [
|
|
3
|
+
"invalid",
|
|
4
|
+
"forbidden",
|
|
5
|
+
"quota",
|
|
6
|
+
"unavailable"
|
|
7
|
+
];
|
|
8
|
+
var QueueError = class extends Error {
|
|
9
|
+
code;
|
|
10
|
+
constructor(code, why) {
|
|
11
|
+
super(why);
|
|
12
|
+
this.code = code;
|
|
13
|
+
this.name = "QueueError";
|
|
14
|
+
}
|
|
15
|
+
};
|
|
16
|
+
/** The suffix that marks a Queue's dead-letter Queue; no Queue's own name has a dot. */
|
|
17
|
+
const DEAD = ".dead";
|
|
18
|
+
/** The delay before a Message's next delivery after its `attempt`-th: 10 seconds, doubling, at most 10 minutes. */
|
|
19
|
+
function backoffMs(attempt) {
|
|
20
|
+
return Math.min(1e4 * 2 ** Math.max(0, attempt - 1), 6e5);
|
|
21
|
+
}
|
|
22
|
+
/** Stats' counts stop at this many. */
|
|
23
|
+
const STATS_CAP = 1e3;
|
|
24
|
+
function invalid(why) {
|
|
25
|
+
throw new QueueError("invalid", why);
|
|
26
|
+
}
|
|
27
|
+
const whole = (n, low, high) => typeof n === "number" && Number.isSafeInteger(n) && n >= low && n <= high;
|
|
28
|
+
function checkQueueName(name) {
|
|
29
|
+
if (!/^[a-z0-9][a-z0-9_-]{0,62}$/.test(name)) invalid(`"${name}" is not a Queue name: 1 to 63 lowercase letters, digits, hyphens and underscores, starting with a letter or digit`);
|
|
30
|
+
}
|
|
31
|
+
/** A name as an engine is given it: a Queue's, or its dead-letter Queue's. */
|
|
32
|
+
function checkEngineQueue(queue) {
|
|
33
|
+
checkQueueName(queue.endsWith(".dead") ? queue.slice(0, -5) : queue);
|
|
34
|
+
}
|
|
35
|
+
function checkKey(key) {
|
|
36
|
+
if (!/^[\x21-\x7e]{1,128}$/.test(key)) invalid("a send's key is 1 to 128 printable characters, without spaces");
|
|
37
|
+
}
|
|
38
|
+
/** Bodies are at most this many bytes of JSON. */
|
|
39
|
+
const MAX_BODY = 131072;
|
|
40
|
+
const encoder = new TextEncoder();
|
|
41
|
+
/** The JSON text of `body`, which must be a JSON value of at most 128 KiB. */
|
|
42
|
+
function bodyText(body) {
|
|
43
|
+
let text;
|
|
44
|
+
try {
|
|
45
|
+
text = JSON.stringify(body);
|
|
46
|
+
} catch {
|
|
47
|
+
invalid("the body is not a JSON value");
|
|
48
|
+
}
|
|
49
|
+
if (text === void 0) invalid("the body is not a JSON value");
|
|
50
|
+
const bytes = encoder.encode(text).length;
|
|
51
|
+
if (bytes > 131072) invalid(`the body is ${bytes} bytes of JSON; at most 131,072`);
|
|
52
|
+
return text;
|
|
53
|
+
}
|
|
54
|
+
function checkReceipt(receipt) {
|
|
55
|
+
if (!/^[A-Za-z0-9._-]{1,200}$/.test(receipt)) invalid("that is not a receipt");
|
|
56
|
+
}
|
|
57
|
+
function checkDelaySeconds(seconds) {
|
|
58
|
+
if (!whole(seconds, 0, 43200)) invalid("a delay is a whole number of seconds, 0 to 43,200");
|
|
59
|
+
}
|
|
60
|
+
function checkReceive(options) {
|
|
61
|
+
if (!whole(options.limit, 1, 100)) invalid("a receive's limit is 1 to 100");
|
|
62
|
+
if (!whole(options.visibilityMs, 1e3, 432e5) || options.visibilityMs % 1e3 !== 0) invalid("a visibility window is a whole number of seconds, 1 to 43,200");
|
|
63
|
+
if (!whole(options.maxAttempts, 1, 100)) invalid("maxAttempts is 1 to 100");
|
|
64
|
+
}
|
|
65
|
+
/** Queues over `engine`. */
|
|
66
|
+
function queues(engine) {
|
|
67
|
+
const queueOf = (name, engineName, dead) => {
|
|
68
|
+
const queue = {
|
|
69
|
+
name,
|
|
70
|
+
async send(body, options = {}) {
|
|
71
|
+
const { ids } = await queue.sendBatch([{
|
|
72
|
+
body,
|
|
73
|
+
...options
|
|
74
|
+
}]);
|
|
75
|
+
return { id: ids[0] };
|
|
76
|
+
},
|
|
77
|
+
async sendBatch(messages) {
|
|
78
|
+
if (messages.length < 1 || messages.length > 100) invalid("a batch sends 1 to 100 Messages");
|
|
79
|
+
const checked = messages.map(({ body, delaySeconds = 0, key }) => {
|
|
80
|
+
checkDelaySeconds(delaySeconds);
|
|
81
|
+
if (key !== void 0) checkKey(key);
|
|
82
|
+
return {
|
|
83
|
+
body: bodyText(body),
|
|
84
|
+
delayMs: delaySeconds * 1e3,
|
|
85
|
+
...key !== void 0 ? { key } : {}
|
|
86
|
+
};
|
|
87
|
+
});
|
|
88
|
+
return { ids: await engine.send(engineName, checked) };
|
|
89
|
+
},
|
|
90
|
+
async receive(options = {}) {
|
|
91
|
+
const call = {
|
|
92
|
+
limit: options.limit ?? 10,
|
|
93
|
+
visibilityMs: (options.visibilitySeconds ?? 30) * 1e3,
|
|
94
|
+
maxAttempts: options.maxAttempts ?? 5
|
|
95
|
+
};
|
|
96
|
+
checkReceive(call);
|
|
97
|
+
return (await engine.receive(engineName, call)).map((delivery) => ({
|
|
98
|
+
...delivery,
|
|
99
|
+
body: JSON.parse(delivery.body)
|
|
100
|
+
}));
|
|
101
|
+
},
|
|
102
|
+
async ack(receipt) {
|
|
103
|
+
checkReceipt(receipt);
|
|
104
|
+
return engine.ack(engineName, receipt);
|
|
105
|
+
},
|
|
106
|
+
async retry(receipt, options = {}) {
|
|
107
|
+
checkReceipt(receipt);
|
|
108
|
+
if (options.delaySeconds !== void 0) checkDelaySeconds(options.delaySeconds);
|
|
109
|
+
return engine.retry(engineName, receipt, options.delaySeconds === void 0 ? void 0 : options.delaySeconds * 1e3);
|
|
110
|
+
},
|
|
111
|
+
stats() {
|
|
112
|
+
return engine.stats(engineName);
|
|
113
|
+
},
|
|
114
|
+
get deadLetter() {
|
|
115
|
+
return dead ?? queue;
|
|
116
|
+
}
|
|
117
|
+
};
|
|
118
|
+
return queue;
|
|
119
|
+
};
|
|
120
|
+
return { queue(name) {
|
|
121
|
+
checkQueueName(name);
|
|
122
|
+
return queueOf(name, name, queueOf(`${name}${DEAD}`, `${name}${DEAD}`));
|
|
123
|
+
} };
|
|
124
|
+
}
|
|
125
|
+
//#endregion
|
|
126
|
+
export { DEAD, MAX_BODY, QueueError, STATS_CAP, backoffMs, bodyText, checkDelaySeconds, checkEngineQueue, checkKey, checkQueueName, checkReceipt, checkReceive, invalid, queueErrorCodes, queues };
|
|
@@ -0,0 +1,110 @@
|
|
|
1
|
+
import { DEAD, STATS_CAP, backoffMs } from "./index.mjs";
|
|
2
|
+
//#region src/queues/stand-in.ts
|
|
3
|
+
/**
|
|
4
|
+
* Queues' Stand-in: the Capability's behaviour in memory, for `midsummer dev` and tests. It holds no
|
|
5
|
+
* engine code and passes the contract (`@midsummer-ai/sdk/queues/contract`) except durability and scale.
|
|
6
|
+
*
|
|
7
|
+
* const jobs = queues(standIn({ clock: Date.now }))
|
|
8
|
+
*
|
|
9
|
+
* A Queue exists from its first send. Everything is lost when the process ends.
|
|
10
|
+
*/
|
|
11
|
+
/** Keys a keyed send is remembered for. */
|
|
12
|
+
const KEY_MS = 864e5;
|
|
13
|
+
function standIn(options) {
|
|
14
|
+
const { clock } = options;
|
|
15
|
+
const queues = /* @__PURE__ */ new Map();
|
|
16
|
+
const keys = /* @__PURE__ */ new Map();
|
|
17
|
+
let sequence = 0;
|
|
18
|
+
const queueOf = (name) => {
|
|
19
|
+
let held = queues.get(name);
|
|
20
|
+
if (!held) queues.set(name, held = []);
|
|
21
|
+
return held;
|
|
22
|
+
};
|
|
23
|
+
const order = (a, b) => a.visibleAt - b.visibleAt || (a.id < b.id ? -1 : a.id > b.id ? 1 : 0);
|
|
24
|
+
const nextId = (at) => `${at.toString(36).padStart(9, "0")}${(sequence++).toString(36).padStart(6, "0")}`;
|
|
25
|
+
/** Moves `held` to its dead-letter Queue when its last delivery was its last; returns whether it did. */
|
|
26
|
+
const deadLetter = (queue, held) => {
|
|
27
|
+
if (queue.endsWith(".dead") || held.maxAttempts === void 0 || held.attempts < held.maxAttempts) return false;
|
|
28
|
+
const list = queueOf(queue);
|
|
29
|
+
list.splice(list.indexOf(held), 1);
|
|
30
|
+
const { receipt: _receipt, maxAttempts: _max, ...rest } = held;
|
|
31
|
+
queueOf(`${queue}${DEAD}`).push({
|
|
32
|
+
...rest,
|
|
33
|
+
visibleAt: clock()
|
|
34
|
+
});
|
|
35
|
+
return true;
|
|
36
|
+
};
|
|
37
|
+
/** The Message a receipt still names: none once its window has ended, though it was not delivered again yet. */
|
|
38
|
+
const delivered = (queue, receipt) => queues.get(queue)?.find((held) => held.receipt === receipt && held.visibleAt > clock());
|
|
39
|
+
return {
|
|
40
|
+
async send(queue, messages) {
|
|
41
|
+
const now = clock();
|
|
42
|
+
return messages.map((message) => {
|
|
43
|
+
const remembered = message.key === void 0 ? void 0 : keys.get(`${queue}\n${message.key}`);
|
|
44
|
+
if (remembered && now - remembered.at < KEY_MS) return remembered.id;
|
|
45
|
+
const id = nextId(now);
|
|
46
|
+
queueOf(queue).push({
|
|
47
|
+
id,
|
|
48
|
+
body: message.body,
|
|
49
|
+
sentAt: now,
|
|
50
|
+
visibleAt: now + message.delayMs,
|
|
51
|
+
attempts: 0
|
|
52
|
+
});
|
|
53
|
+
if (message.key !== void 0) keys.set(`${queue}\n${message.key}`, {
|
|
54
|
+
id,
|
|
55
|
+
at: now
|
|
56
|
+
});
|
|
57
|
+
return id;
|
|
58
|
+
});
|
|
59
|
+
},
|
|
60
|
+
async receive(queue, { limit, visibilityMs, maxAttempts }) {
|
|
61
|
+
const now = clock();
|
|
62
|
+
const out = [];
|
|
63
|
+
for (const held of [...queues.get(queue) ?? []].sort(order)) {
|
|
64
|
+
if (out.length === limit || held.visibleAt > now) break;
|
|
65
|
+
if (held.receipt !== void 0 && deadLetter(queue, held)) continue;
|
|
66
|
+
held.attempts += 1;
|
|
67
|
+
held.visibleAt = now + visibilityMs;
|
|
68
|
+
held.receipt = `${held.id}.${crypto.randomUUID().replaceAll("-", "")}`;
|
|
69
|
+
held.maxAttempts = maxAttempts;
|
|
70
|
+
out.push({
|
|
71
|
+
id: held.id,
|
|
72
|
+
body: held.body,
|
|
73
|
+
sentAt: held.sentAt,
|
|
74
|
+
attempt: held.attempts,
|
|
75
|
+
receipt: held.receipt,
|
|
76
|
+
visibleUntil: held.visibleAt
|
|
77
|
+
});
|
|
78
|
+
}
|
|
79
|
+
return out;
|
|
80
|
+
},
|
|
81
|
+
async ack(queue, receipt) {
|
|
82
|
+
const held = delivered(queue, receipt);
|
|
83
|
+
if (held) queueOf(queue).splice(queueOf(queue).indexOf(held), 1);
|
|
84
|
+
},
|
|
85
|
+
async retry(queue, receipt, delayMs) {
|
|
86
|
+
const held = delivered(queue, receipt);
|
|
87
|
+
if (!held || deadLetter(queue, held)) return;
|
|
88
|
+
held.visibleAt = clock() + (delayMs ?? backoffMs(held.attempts));
|
|
89
|
+
delete held.receipt;
|
|
90
|
+
delete held.maxAttempts;
|
|
91
|
+
},
|
|
92
|
+
async stats(queue) {
|
|
93
|
+
const now = clock();
|
|
94
|
+
const held = queues.get(queue) ?? [];
|
|
95
|
+
const counted = [...held].sort((a, b) => a.sentAt - b.sentAt).slice(0, STATS_CAP);
|
|
96
|
+
const dead = (queues.get(`${queue}.dead`) ?? []).length;
|
|
97
|
+
return {
|
|
98
|
+
depth: counted.length,
|
|
99
|
+
ready: counted.filter((m) => m.visibleAt <= now).length,
|
|
100
|
+
inFlight: counted.filter((m) => m.receipt !== void 0 && m.visibleAt > now).length,
|
|
101
|
+
delayed: counted.filter((m) => m.receipt === void 0 && m.visibleAt > now).length,
|
|
102
|
+
...counted[0] ? { oldestSentAt: counted[0].sentAt } : {},
|
|
103
|
+
dead: Math.min(dead, STATS_CAP),
|
|
104
|
+
...held.length > 1e3 || dead > 1e3 ? { more: true } : {}
|
|
105
|
+
};
|
|
106
|
+
}
|
|
107
|
+
};
|
|
108
|
+
}
|
|
109
|
+
//#endregion
|
|
110
|
+
export { standIn };
|
|
@@ -0,0 +1,9 @@
|
|
|
1
|
+
import { Transport } from "../wire.mjs";
|
|
2
|
+
import { QueueEngine } from "./index.mjs";
|
|
3
|
+
//#region src/queues/wire.d.ts
|
|
4
|
+
/** The engine on the far side of `transport`. */
|
|
5
|
+
declare function remote(transport: Transport): QueueEngine;
|
|
6
|
+
/** Answers frames from `remote` with `engine`, checking each again: a frame comes from the network. */
|
|
7
|
+
declare function serve(engine: QueueEngine): (frame: Uint8Array) => Promise<Uint8Array>;
|
|
8
|
+
//#endregion
|
|
9
|
+
export { type Transport, remote, serve };
|
package/queues/wire.mjs
ADDED
|
@@ -0,0 +1,199 @@
|
|
|
1
|
+
import { decodeFrame, encodeFrame } from "../wire.mjs";
|
|
2
|
+
import { QueueError, checkEngineQueue, checkKey, checkReceipt, checkReceive, queueErrorCodes } from "./index.mjs";
|
|
3
|
+
import { z } from "zod";
|
|
4
|
+
//#region src/queues/wire.ts
|
|
5
|
+
/**
|
|
6
|
+
* Queues' wire: how a Queues call crosses a network. `remote(transport)` is the engine a client uses;
|
|
7
|
+
* `serve(engine)` replies on the other side. A server implements `serve` over its own engine and
|
|
8
|
+
* nothing else, so the contract run through the wire holds for it too.
|
|
9
|
+
*
|
|
10
|
+
* const jobs = queues(remote({ send: (frame) => env.QUEUES.send(frame) })) // in a Site
|
|
11
|
+
* handle.onFrame(serve(engine)) // on the platform
|
|
12
|
+
*
|
|
13
|
+
* Frames are Storage's (`@midsummer-ai/sdk/wire`): a 4-byte little-endian length and that many bytes of
|
|
14
|
+
* JSON header, here with no body after it. One request frame gets one answer frame. Changing a frame
|
|
15
|
+
* changes the contract with every server.
|
|
16
|
+
*/
|
|
17
|
+
const request = z.discriminatedUnion("op", [
|
|
18
|
+
z.object({
|
|
19
|
+
op: z.literal("queue.send"),
|
|
20
|
+
queue: z.string(),
|
|
21
|
+
messages: z.array(z.object({
|
|
22
|
+
body: z.string(),
|
|
23
|
+
delayMs: z.number(),
|
|
24
|
+
key: z.string().optional()
|
|
25
|
+
}).strict()).min(1).max(100)
|
|
26
|
+
}).strict(),
|
|
27
|
+
z.object({
|
|
28
|
+
op: z.literal("queue.receive"),
|
|
29
|
+
queue: z.string(),
|
|
30
|
+
limit: z.number(),
|
|
31
|
+
visibilityMs: z.number(),
|
|
32
|
+
maxAttempts: z.number()
|
|
33
|
+
}).strict(),
|
|
34
|
+
z.object({
|
|
35
|
+
op: z.literal("queue.ack"),
|
|
36
|
+
queue: z.string(),
|
|
37
|
+
receipt: z.string()
|
|
38
|
+
}).strict(),
|
|
39
|
+
z.object({
|
|
40
|
+
op: z.literal("queue.retry"),
|
|
41
|
+
queue: z.string(),
|
|
42
|
+
receipt: z.string(),
|
|
43
|
+
delayMs: z.number().optional()
|
|
44
|
+
}).strict(),
|
|
45
|
+
z.object({
|
|
46
|
+
op: z.literal("queue.stats"),
|
|
47
|
+
queue: z.string()
|
|
48
|
+
}).strict()
|
|
49
|
+
]);
|
|
50
|
+
const delivery = z.object({
|
|
51
|
+
id: z.string(),
|
|
52
|
+
body: z.string(),
|
|
53
|
+
sentAt: z.number(),
|
|
54
|
+
attempt: z.number(),
|
|
55
|
+
receipt: z.string(),
|
|
56
|
+
visibleUntil: z.number()
|
|
57
|
+
}).strict();
|
|
58
|
+
const stats = z.object({
|
|
59
|
+
depth: z.number(),
|
|
60
|
+
ready: z.number(),
|
|
61
|
+
inFlight: z.number(),
|
|
62
|
+
delayed: z.number(),
|
|
63
|
+
oldestSentAt: z.number().optional(),
|
|
64
|
+
dead: z.number(),
|
|
65
|
+
more: z.literal(true).optional()
|
|
66
|
+
}).strict();
|
|
67
|
+
const answer = z.union([
|
|
68
|
+
z.object({
|
|
69
|
+
error: z.enum(queueErrorCodes),
|
|
70
|
+
why: z.string()
|
|
71
|
+
}).strict(),
|
|
72
|
+
z.object({ ids: z.array(z.string()) }).strict(),
|
|
73
|
+
z.object({ deliveries: z.array(delivery) }).strict(),
|
|
74
|
+
z.object({ stats }).strict(),
|
|
75
|
+
z.object({ done: z.literal(true) }).strict()
|
|
76
|
+
]);
|
|
77
|
+
/** The engine on the far side of `transport`. */
|
|
78
|
+
function remote(transport) {
|
|
79
|
+
const call = async (header) => {
|
|
80
|
+
let frame;
|
|
81
|
+
try {
|
|
82
|
+
frame = await transport.send(encodeFrame(header));
|
|
83
|
+
} catch (error) {
|
|
84
|
+
if (error instanceof QueueError) throw error;
|
|
85
|
+
if (error instanceof Error && "code" in error && (error.code === "forbidden" || error.code === "unavailable")) throw new QueueError(error.code, "Queues could not be reached");
|
|
86
|
+
throw new QueueError("unavailable", "Queues could not be reached");
|
|
87
|
+
}
|
|
88
|
+
let parsed;
|
|
89
|
+
try {
|
|
90
|
+
parsed = answer.parse(decodeFrame(frame).header);
|
|
91
|
+
} catch {
|
|
92
|
+
throw new QueueError("unavailable", "Queues answered something that is not a Queues answer");
|
|
93
|
+
}
|
|
94
|
+
if ("error" in parsed) throw new QueueError(parsed.error, parsed.why);
|
|
95
|
+
return parsed;
|
|
96
|
+
};
|
|
97
|
+
const expect = (answered, key) => {
|
|
98
|
+
if (!(key in answered)) throw new QueueError("unavailable", "Queues answered another call");
|
|
99
|
+
return answered[key];
|
|
100
|
+
};
|
|
101
|
+
return {
|
|
102
|
+
async send(queue, messages) {
|
|
103
|
+
return expect(await call({
|
|
104
|
+
op: "queue.send",
|
|
105
|
+
queue,
|
|
106
|
+
messages: [...messages]
|
|
107
|
+
}), "ids");
|
|
108
|
+
},
|
|
109
|
+
async receive(queue, options) {
|
|
110
|
+
return expect(await call({
|
|
111
|
+
op: "queue.receive",
|
|
112
|
+
queue,
|
|
113
|
+
...options
|
|
114
|
+
}), "deliveries");
|
|
115
|
+
},
|
|
116
|
+
async ack(queue, receipt) {
|
|
117
|
+
await call({
|
|
118
|
+
op: "queue.ack",
|
|
119
|
+
queue,
|
|
120
|
+
receipt
|
|
121
|
+
});
|
|
122
|
+
},
|
|
123
|
+
async retry(queue, receipt, delayMs) {
|
|
124
|
+
await call({
|
|
125
|
+
op: "queue.retry",
|
|
126
|
+
queue,
|
|
127
|
+
receipt,
|
|
128
|
+
...delayMs === void 0 ? {} : { delayMs }
|
|
129
|
+
});
|
|
130
|
+
},
|
|
131
|
+
async stats(queue) {
|
|
132
|
+
return expect(await call({
|
|
133
|
+
op: "queue.stats",
|
|
134
|
+
queue
|
|
135
|
+
}), "stats");
|
|
136
|
+
}
|
|
137
|
+
};
|
|
138
|
+
}
|
|
139
|
+
const encoder = new TextEncoder();
|
|
140
|
+
/** Answers frames from `remote` with `engine`, checking each again: a frame comes from the network. */
|
|
141
|
+
function serve(engine) {
|
|
142
|
+
return async (frame) => {
|
|
143
|
+
try {
|
|
144
|
+
let header;
|
|
145
|
+
try {
|
|
146
|
+
header = decodeFrame(frame).header;
|
|
147
|
+
} catch {
|
|
148
|
+
throw new QueueError("invalid", "the frame is not a frame");
|
|
149
|
+
}
|
|
150
|
+
const parsed = request.safeParse(header);
|
|
151
|
+
if (!parsed.success) throw new QueueError("invalid", "the request frame is not a Queues call");
|
|
152
|
+
const call = parsed.data;
|
|
153
|
+
checkEngineQueue(call.queue);
|
|
154
|
+
switch (call.op) {
|
|
155
|
+
case "queue.send":
|
|
156
|
+
for (const message of call.messages) {
|
|
157
|
+
if (message.key !== void 0) checkKey(message.key);
|
|
158
|
+
if (!Number.isSafeInteger(message.delayMs) || message.delayMs < 0 || message.delayMs > 432e5) throw new QueueError("invalid", "a delay is 0 to 43,200 seconds");
|
|
159
|
+
if (encoder.encode(message.body).length > 131072) throw new QueueError("invalid", "a body is at most 131,072 bytes of JSON");
|
|
160
|
+
try {
|
|
161
|
+
JSON.parse(message.body);
|
|
162
|
+
} catch {
|
|
163
|
+
throw new QueueError("invalid", "a body is not JSON");
|
|
164
|
+
}
|
|
165
|
+
}
|
|
166
|
+
return encodeFrame({ ids: await engine.send(call.queue, call.messages.map(({ body, delayMs, key }) => ({
|
|
167
|
+
body,
|
|
168
|
+
delayMs,
|
|
169
|
+
...key === void 0 ? {} : { key }
|
|
170
|
+
}))) });
|
|
171
|
+
case "queue.receive":
|
|
172
|
+
checkReceive(call);
|
|
173
|
+
return encodeFrame({ deliveries: await engine.receive(call.queue, {
|
|
174
|
+
limit: call.limit,
|
|
175
|
+
visibilityMs: call.visibilityMs,
|
|
176
|
+
maxAttempts: call.maxAttempts
|
|
177
|
+
}) });
|
|
178
|
+
case "queue.ack":
|
|
179
|
+
checkReceipt(call.receipt);
|
|
180
|
+
await engine.ack(call.queue, call.receipt);
|
|
181
|
+
return encodeFrame({ done: true });
|
|
182
|
+
case "queue.retry":
|
|
183
|
+
checkReceipt(call.receipt);
|
|
184
|
+
if (call.delayMs !== void 0 && (!Number.isSafeInteger(call.delayMs) || call.delayMs < 0 || call.delayMs > 432e5)) throw new QueueError("invalid", "a delay is 0 to 43,200 seconds");
|
|
185
|
+
await engine.retry(call.queue, call.receipt, call.delayMs);
|
|
186
|
+
return encodeFrame({ done: true });
|
|
187
|
+
case "queue.stats": return encodeFrame({ stats: await engine.stats(call.queue) });
|
|
188
|
+
}
|
|
189
|
+
} catch (error) {
|
|
190
|
+
if (error instanceof QueueError) return encodeFrame({
|
|
191
|
+
error: error.code,
|
|
192
|
+
why: error.message
|
|
193
|
+
});
|
|
194
|
+
throw error;
|
|
195
|
+
}
|
|
196
|
+
};
|
|
197
|
+
}
|
|
198
|
+
//#endregion
|
|
199
|
+
export { remote, serve };
|
package/s3.d.mts
ADDED
|
@@ -0,0 +1,43 @@
|
|
|
1
|
+
import { m as StorageEngine } from "./index-sgdyoA2P.mjs";
|
|
2
|
+
//#region src/s3.d.ts
|
|
3
|
+
/** What one access key may do. */
|
|
4
|
+
interface S3Credential {
|
|
5
|
+
/** The secret access key that signs its requests. */
|
|
6
|
+
readonly secret: string;
|
|
7
|
+
/** `read`: list, get and head. `write`: everything. */
|
|
8
|
+
readonly access: "read" | "write";
|
|
9
|
+
/**
|
|
10
|
+
* The Buckets it may use, by name, each with the name the engine knows it by. When not set, every
|
|
11
|
+
* Bucket, under its own name, as under `midsummer dev`.
|
|
12
|
+
*/
|
|
13
|
+
readonly buckets?: Readonly<Record<string, string>>;
|
|
14
|
+
/** Buckets, by name, whose writes are paused (their Team reached its budget): reads go on, writes are `QuotaExceeded`. */
|
|
15
|
+
readonly paused?: ReadonlyArray<string>;
|
|
16
|
+
}
|
|
17
|
+
interface S3Options {
|
|
18
|
+
/**
|
|
19
|
+
* The credential of an access key id, or `null` for one that is unknown or revoked. `named` is the
|
|
20
|
+
* Buckets the request names (its own and a copy's source); the credential may then answer only
|
|
21
|
+
* those of its `buckets`. Without it (a request listing the key's Buckets), every one.
|
|
22
|
+
*/
|
|
23
|
+
credentials(accessKeyId: string, named?: ReadonlyArray<string>): S3Credential | null | Promise<S3Credential | null>;
|
|
24
|
+
/** Hostnames whose subdomains are Buckets, as `avatars.<host>`. Path-style addressing always works. */
|
|
25
|
+
readonly hosts?: ReadonlyArray<string>;
|
|
26
|
+
/**
|
|
27
|
+
* The path the handler is mounted at, such as `/s3`: Buckets are addressed after it, and clients
|
|
28
|
+
* take it as part of their endpoint (`https://api.example.com/s3`). Signatures cover the whole path.
|
|
29
|
+
*/
|
|
30
|
+
readonly base?: string;
|
|
31
|
+
readonly clock: () => number;
|
|
32
|
+
/**
|
|
33
|
+
* Told, once a Bucket has served a request, the name the engine knows that Bucket by and the class
|
|
34
|
+
* S3 bills the request in. A request that is not signed, is refused access, is replied to
|
|
35
|
+
* `NotImplemented`, deletes (deletes are free), or names no Bucket is not counted.
|
|
36
|
+
*/
|
|
37
|
+
readonly counted?: (bucket: string, kind: RequestClass) => void;
|
|
38
|
+
}
|
|
39
|
+
/** How S3 bills a request: `a` for what writes or lists (PUT, POST, COPY, LIST), `b` for what reads (GET, HEAD). */
|
|
40
|
+
type RequestClass = "a" | "b";
|
|
41
|
+
declare function s3(engine: StorageEngine, options: S3Options): (request: Request) => Promise<Response>;
|
|
42
|
+
//#endregion
|
|
43
|
+
export { RequestClass, S3Credential, S3Options, s3 };
|