@mark1russell7/service 0.1.0 → 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +120 -15
- package/build/builder-BFxOuULH.d.ts +607 -0
- package/build/clock-DlJFFZlX.js +39 -0
- package/build/clock-DlJFFZlX.js.map +1 -0
- package/build/index.d.ts +47 -228
- package/build/index.js +1536 -88
- package/build/index.js.map +1 -1
- package/build/redis.d.ts +294 -0
- package/build/redis.js +853 -0
- package/build/redis.js.map +1 -0
- package/build/testing.d.ts +102 -0
- package/build/testing.js +349 -0
- package/build/testing.js.map +1 -0
- package/build/timing-DxZ4nkQg.js +79 -0
- package/build/timing-DxZ4nkQg.js.map +1 -0
- package/package.json +20 -2
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"redis.js","names":["dialTcp"],"sources":["../src/redis/connection.ts","../src/redis/meets.ts","../src/redis/operations.ts","../src/redis/client.ts","../src/redis/machine.ts"],"sourcesContent":["/**\n * A Redis connection is the part of an ioredis client that the wrapper uses. An ioredis 6 client has\n * each part. A fake for tests has them too. Thus the package depends on no type of ioredis.\n */\nexport interface RedisConnection {\n /** This text is the state of the client: wait, connecting, connect, ready, close, reconnecting or end. */\n readonly status: string;\n /** This socket is the socket of the current connection, when there is one. */\n readonly stream?: RedisSocket | undefined;\n connect(): Promise<void>;\n /** This method sends one command, and gives the reply. */\n call(command: string, ...args: Array<string | number>): Promise<unknown>;\n quit(): Promise<unknown>;\n /** This method closes the connection at once. With `true`, the client then connects again. */\n disconnect(reconnect?: boolean): void;\n on(event: string, listener: (...args: unknown[]) => void): unknown;\n off(event: string, listener: (...args: unknown[]) => void): unknown;\n}\n\n/** A Redis socket is the part of the socket of a connection that the join of the prober reads. */\nexport interface RedisSocket {\n readonly localAddress?: string | undefined;\n readonly localPort?: number | undefined;\n readonly remoteAddress?: string | undefined;\n readonly remotePort?: number | undefined;\n /** This method closes the socket at once. A disconnect only half-closes it, and a silent peer does not answer that. */\n destroy?(): void;\n}\n\n/**\n * These are the options that the wrapper gives ioredis. Each one comes from the declaration. The\n * wrapper sets each option that has an effect on a timer or on a send again, thus no default of\n * ioredis decides one.\n */\nexport interface RedisConnectionOptions {\n readonly host: string;\n readonly port: number;\n readonly db?: number | undefined;\n readonly username?: string | undefined;\n readonly password?: string | undefined;\n /** The wrapper connects in its own step of the startup. */\n readonly lazyConnect: true;\n readonly connectTimeout: number;\n readonly commandTimeout: number;\n /** This time destroys a socket that gives no data while a command waits. Undefined is no bound. */\n readonly socketTimeout: number | undefined;\n /** This time is the idle time of TCP keepalive. False is no keepalive. */\n readonly keepAlive: number | false;\n /** A command while the client is not ready fails at once: it did not reach the server. */\n readonly enableOfflineQueue: false;\n /** The client does not send again a command that can have reached the server. */\n readonly autoResendUnfulfilledCommands: false;\n readonly maxRetriesPerRequest: null;\n readonly enableReadyCheck: true;\n readonly connectionName: string;\n /** This function gives the delay before the next try to connect, or null to stop. */\n readonly retryStrategy: (times: number) => number | null;\n}\n\n/** A connection factory makes the connection of the client from its options. */\nexport type ConnectionFactory = (options: RedisConnectionOptions) => RedisConnection | Promise<RedisConnection>;\n\n/**\n * This factory makes an ioredis client. It loads ioredis only when a client connects, thus a\n * service that does not use Redis does not need ioredis.\n */\nexport const ioredis: ConnectionFactory = async (options) => {\n const loaded = (await import(\"ioredis\")) as unknown as {\n readonly default: new (options: RedisConnectionOptions) => RedisConnection;\n };\n return new loaded.default(options);\n};\n","import type { Extent, Quantity } from \"@mark1russell7/quantity\";\nimport { quantity, valueIn } from \"@mark1russell7/quantity\";\nimport type { RedisKnobs } from \"./client.ts\";\n\n/** A situation is what the connection did when the failure started: a command in flight, or idle. */\nexport type Situation = \"in flight\" | \"idle\";\n\n/** A detector is a mechanism that finds a failure, and the most time that it takes. */\nexport interface Detector {\n /** This name identifies the mechanism: a knob of the declaration, or the kernel. */\n readonly by: string;\n /** This time is the most that the detector takes, after the failure starts. \"infinite\" tells that it does not find the failure. */\n readonly within: Extent;\n readonly situation: Situation;\n /** This text tells what the detector does, for example that it fails a command without a change of state. */\n readonly effect: string;\n}\n\n/**\n * A meeting is a manifestation that the client can meet, and its detectors (E9). The names of the\n * manifestations are the names of the vocabulary of the harness and the corpus. `app-stall` is a\n * server whose process is alive and does not answer, a proposal of the vocabulary.\n */\nexport interface Meeting {\n readonly manifestation: string;\n /** This text tells the failure in words. */\n readonly what: string;\n readonly detectors: readonly Detector[];\n}\n\nconst never: Extent = { value: \"infinite\", unit: \"s\" };\nconst s = (value: number): Quantity => quantity(Math.round(value * 1e6) / 1e6, \"s\");\nconst seconds = (amount: Extent): number =>\n amount.value === \"infinite\" ? Number.POSITIVE_INFINITY : valueIn(amount as Quantity, \"s\");\nconst plus = (amount: Extent, more: number): Extent =>\n amount.value === \"infinite\" ? never : s(seconds(amount) + more);\n\n/** Node gives keepalive 10 probes, 1 s apart, after its idle time: a silent peer is found 10 s after the idle time. */\nconst NODE_PROBES = 10;\nconst NODE_INTERVAL = 1;\n\n/**\n * This function gives the manifestations that a Redis client can meet, and the detectors of each, from\n * its knobs. A reset or a FIN arrives on the next segment. A black hole and a host that is gone give\n * no answer at all. A frozen process and a stalled server keep a kernel that acknowledges each\n * segment. Thus TCP keepalive does not fire on them, and only the client's own timers find them.\n */\nexport const meetings = (name: string, knobs: RedisKnobs, heartbeat: Extent): Meeting[] => {\n const keepalive = plus(knobs.keepalive_idle, NODE_PROBES * NODE_INTERVAL);\n const watchdog = (situation: Situation): Detector => ({\n by: \"heartbeat_detection_time\",\n within: heartbeat,\n situation,\n effect: `the watchdog of ${name} closes the connection, and the client connects again`,\n });\n const writeoff: Detector = {\n by: \"driver_writeoff_time\",\n within: knobs.driver_writeoff_time,\n situation: \"in flight\",\n effect: \"ioredis destroys the socket of a command without data, and the client connects again\",\n };\n const command: Detector = {\n by: \"request_timeout\",\n within: knobs.request_timeout,\n situation: \"in flight\",\n effect: \"the command fails as indeterminate, and the state of the connection does not change\",\n };\n const giveUp: Detector = {\n by: \"retrans_giveup_time\",\n within: knobs.retrans_giveup_time,\n situation: \"in flight\",\n effect: \"the kernel gives up the retransmissions and closes the socket\",\n };\n const silent = (manifestation: string, what: string): Meeting => ({\n manifestation,\n what,\n detectors: [\n command,\n writeoff,\n watchdog(\"in flight\"),\n giveUp,\n watchdog(\"idle\"),\n { by: \"keepalive_idle\", within: keepalive, situation: \"idle\", effect: \"TCP keepalive gives up, and the socket closes\" },\n ],\n });\n const acknowledged = (manifestation: string, what: string): Meeting => ({\n manifestation,\n what,\n detectors: [\n command,\n writeoff,\n watchdog(\"in flight\"),\n watchdog(\"idle\"),\n { by: \"keepalive_idle\", within: never, situation: \"idle\", effect: \"the kernel of the server answers each probe\" },\n ],\n });\n const immediate = (situation: Situation, effect: string): Detector => ({ by: \"the next segment\", within: s(0), situation, effect });\n return [\n {\n manifestation: \"rst\",\n what: \"the server answers each segment with a reset, and refuses each new connection\",\n detectors: [\n immediate(\"in flight\", \"the command fails, and the client connects again, and each attempt is refused\"),\n watchdog(\"idle\"),\n { by: \"keepalive_idle\", within: keepalive.value === \"infinite\" ? never : knobs.keepalive_idle, situation: \"idle\", effect: \"the first probe gets a reset\" },\n ],\n },\n {\n manifestation: \"fin\",\n what: \"the process of the server ends, and its kernel closes the connection\",\n detectors: [immediate(\"in flight\", \"the close arrives, and the command fails\"), immediate(\"idle\", \"the close arrives\")],\n },\n silent(\"blackhole\", \"the network drops each segment in the two directions\"),\n silent(\"freeze-host\", \"the host of the server stops, without a close: the death without a close\"),\n acknowledged(\"freeze-process\", \"the process of the server stops, and its kernel stays up\"),\n acknowledged(\"app-stall\", \"the server stops its replies for a time, for example a long command, and its kernel stays up\"),\n ];\n};\n\n/** This function gives the first detector of a manifestation in a situation, or null when no detector has an end (C3). */\nexport const firstDetector = (meeting: Meeting, situation: Situation): Detector | null => {\n const ranked = meeting.detectors\n .filter((detector) => detector.situation === situation && detector.within.value !== \"infinite\")\n .sort((a, b) => seconds(a.within) - seconds(b.within));\n return ranked[0] ?? null;\n};\n\n/** This function gives each manifestation and situation that no detector of the client finds: the standing rule, as a query (C3, E9). */\nexport const unwatched = (meets: readonly Meeting[]): Array<{ readonly manifestation: string; readonly situation: Situation }> =>\n meets.flatMap((meeting) =>\n ([\"in flight\", \"idle\"] as const).flatMap((situation) =>\n firstDetector(meeting, situation) === null ? [{ manifestation: meeting.manifestation, situation }] : [],\n ),\n );\n","import type { Operation } from \"../operation.ts\";\n\n/**\n * These are the Redis commands whose idempotency the client declares when the declaration does not\n * say. A second issue of an idempotent command has the same effect. Thus the client can send it\n * again after a failure that left its result unknown. The client does not send again a command\n * that is not in this table: a declaration that uses it says its idempotency itself.\n */\nconst SAME_EFFECT = [\n // reads\n \"PING\",\n \"ECHO\",\n \"GET\",\n \"MGET\",\n \"STRLEN\",\n \"GETRANGE\",\n \"EXISTS\",\n \"TYPE\",\n \"TTL\",\n \"PTTL\",\n \"EXPIRETIME\",\n \"HGET\",\n \"HMGET\",\n \"HGETALL\",\n \"HKEYS\",\n \"HVALS\",\n \"HLEN\",\n \"HEXISTS\",\n \"SMEMBERS\",\n \"SISMEMBER\",\n \"SMISMEMBER\",\n \"SCARD\",\n \"ZRANGE\",\n \"ZRANGEBYSCORE\",\n \"ZSCORE\",\n \"ZMSCORE\",\n \"ZRANK\",\n \"ZCARD\",\n \"ZCOUNT\",\n \"LRANGE\",\n \"LINDEX\",\n \"LLEN\",\n \"XRANGE\",\n \"XREVRANGE\",\n \"XLEN\",\n \"SCAN\",\n \"SSCAN\",\n \"HSCAN\",\n \"ZSCAN\",\n \"DBSIZE\",\n \"INFO\",\n \"CLIENT ID\",\n // writes that put a state, not a change of a state: a second issue puts the same state\n \"SET\",\n \"MSET\",\n \"SETEX\",\n \"PSETEX\",\n \"DEL\",\n \"UNLINK\",\n \"EXPIRE\",\n \"PEXPIRE\",\n \"EXPIREAT\",\n \"PEXPIREAT\",\n \"PERSIST\",\n \"HSET\",\n \"HDEL\",\n \"SADD\",\n \"SREM\",\n \"ZADD\",\n \"ZREM\",\n];\n\n/**\n * These commands change a state relative to the state before them. A second issue changes it again:\n * an INCR that the client sends again after a timeout counts two times.\n */\nconst CHANGE = [\n \"INCR\",\n \"INCRBY\",\n \"INCRBYFLOAT\",\n \"DECR\",\n \"DECRBY\",\n \"HINCRBY\",\n \"HINCRBYFLOAT\",\n \"ZINCRBY\",\n \"APPEND\",\n \"LPUSH\",\n \"RPUSH\",\n \"LPOP\",\n \"RPOP\",\n \"BLPOP\",\n \"BRPOP\",\n \"LMOVE\",\n \"BLMOVE\",\n \"SPOP\",\n \"XADD\",\n \"XACK\",\n \"PUBLISH\",\n \"EVAL\",\n \"EVALSHA\",\n \"FCALL\",\n \"MULTI\",\n \"EXEC\",\n \"GETDEL\",\n \"GETSET\",\n \"SETNX\",\n];\n\n/** These are the operations of the client: the idempotency of each command that the table names. */\nexport const REDIS_OPERATIONS: Readonly<Record<string, Operation>> = Object.fromEntries([\n ...SAME_EFFECT.map((command) => [command, { idempotent: true }] as const),\n ...CHANGE.map((command) => [command, { idempotent: false }] as const),\n]);\n\n/** This function gives the name of a command as the table writes it: upper case, and CLIENT ID with its subcommand. */\nexport const commandName = (command: string, args: readonly unknown[]): string => {\n const name = command.toUpperCase();\n if ((name === \"CLIENT\" || name === \"CONFIG\" || name === \"DEBUG\") && typeof args[0] === \"string\")\n return `${name} ${args[0].toUpperCase()}`;\n return name;\n};\n","import { connect as dialTcp } from \"node:net\";\nimport type { Extent, Quantity } from \"@mark1russell7/quantity\";\nimport { quantity, valueIn } from \"@mark1russell7/quantity\";\nimport type { ServiceBuilder } from \"../builder.ts\";\nimport type { Cancel, Clock } from \"../clock.ts\";\nimport { realClock } from \"../clock.ts\";\nimport type { Check, Dependency, Gate, ServiceDescription, Step } from \"../description.ts\";\nimport type { Recorder } from \"../events.ts\";\nimport type { Outcome } from \"../latch.ts\";\nimport type { DependencyHandle } from \"../lifecycle.ts\";\nimport type { Operation } from \"../operation.ts\";\nimport { mayReissue } from \"../operation.ts\";\nimport type { CheckFn } from \"../poller.ts\";\nimport type { Poll } from \"../timing.ts\";\nimport type { ConnectionFactory, RedisConnection, RedisConnectionOptions } from \"./connection.ts\";\nimport { ioredis } from \"./connection.ts\";\nimport type { Meeting } from \"./meets.ts\";\nimport { meetings } from \"./meets.ts\";\nimport { REDIS_OPERATIONS, commandName } from \"./operations.ts\";\n\n/**\n * These are the knobs of the client, each a time, by the names of the ordering law where the law\n * has them. The runtime has no defaults, thus each knob is necessary, and a bound that nobody gives\n * is \"infinite\", not zero.\n */\nexport interface RedisKnobs {\n /** This time bounds a try to connect, until the TCP connection exists (`connectTimeout`). */\n readonly connect_bound: Quantity;\n /** This time bounds each command, the handshake commands too (`commandTimeout`). */\n readonly request_timeout: Quantity;\n /** This time destroys a socket that gives no data while a command waits (`socketTimeout`). */\n readonly driver_writeoff_time: Extent;\n /** This time is the idle time of TCP keepalive. Node then sends 10 probes 1 s apart. */\n readonly keepalive_idle: Extent;\n /** This time is the interval of the in-band watchdog: a PING on the connection of the application. */\n readonly heartbeat_interval: Extent;\n /** This time bounds each PING of the watchdog. */\n readonly heartbeat_timeout: Quantity;\n /** This time is how long the client tries to connect again before it stops. The watchdog then starts it again. */\n readonly client_retry_window: Extent;\n /** This time is the first delay before a new try to connect. The delay doubles after each try. */\n readonly retry_base: Quantity;\n /** This time is the most that one delay can be, before its jitter. */\n readonly retry_cap: Quantity;\n /** This time bounds the close of the client at the shutdown: a QUIT, then a disconnect. */\n readonly close_bound: Quantity;\n /** This time is the age at which the client replaces its connection. The client supports only \"infinite\". */\n readonly max_connection_age: Extent;\n /**\n * This time is when the kernel gives up a segment that it retransmits. Node cannot set it, thus\n * the declaration states it for the ordering law (O17), from the platform.\n */\n readonly retrans_giveup_time: Extent;\n}\n\n/** A Redis client spec is the declaration of one Redis client: where it connects, and its knobs. */\nexport interface RedisClientSpec {\n readonly host: string;\n readonly port: number;\n readonly db?: number | undefined;\n readonly username?: string | undefined;\n readonly password?: string | undefined;\n /** This identifier is the component in the corpus (`systems.component.redis`), or null. */\n readonly system: string | null;\n readonly knobs: RedisKnobs;\n /** This count is the number of PINGs in sequence without an answer that make the watchdog connect again. */\n readonly heartbeatMisses: number;\n /** These operations add to the table of the client, or change it. */\n readonly operations?: Readonly<Record<string, Operation>> | undefined;\n}\n\n/** A dial function opens a new TCP connection, sends one PING, and reads the reply. The `dial` check uses it. */\nexport type DialFn = (host: string, port: number, signal: AbortSignal) => Promise<void>;\n\n/** These options make a client. Without them, it uses ioredis, the real clock and `Math.random`. */\nexport interface RedisClientOptions {\n readonly create?: ConnectionFactory | undefined;\n readonly clock?: Clock | undefined;\n readonly random?: (() => number) | undefined;\n /** This function receives each transition of the client, for the prober (D3). */\n readonly record?: Recorder | undefined;\n readonly dial?: DialFn | undefined;\n /**\n * This function waits one turn of the event loop. The watchdog waits it before it counts a miss,\n * thus a reply that is in the socket is read first. Without it, the client uses `setImmediate`.\n */\n readonly turn?: (() => Promise<void>) | undefined;\n}\n\n/**\n * A join is one generation of the connection, at its `ready` (D1, D2). The prober joins its kernel\n * records to the server's view of the connection through it. A join holds the server's id of the\n * connection, and the local address and port of the socket.\n */\nexport interface Join {\n readonly generation: number;\n /** This time is when the connection became ready, in milliseconds on the monotonic clock. */\n readonly at: number;\n /** This number is the reply of CLIENT ID, or null when the server gave none. */\n readonly clientId: number | null;\n readonly localAddress: string | null;\n readonly localPort: number | null;\n readonly remoteAddress: string | null;\n readonly remotePort: number | null;\n}\n\n/**\n * A sent result is the outcome of a command, in three values (C8):\n * - `ok`: the server answered.\n * - `error`: the command did not apply. The server answered with an error (`reply`), or the command\n * did not leave the client.\n * - `indeterminate`: the command can have reached the server, and nothing tells if it applied.\n */\nexport type Sent<T> =\n | { readonly outcome: \"ok\"; readonly value: T; readonly attempts: number }\n | { readonly outcome: \"error\"; readonly error: string; readonly reply: boolean; readonly attempts: number }\n | { readonly outcome: \"indeterminate\"; readonly error: string; readonly attempts: number };\n\n/** Send options tell how the client sends one command. */\nexport interface SendOptions {\n /** This signal abandons the command. A command that left the client is then indeterminate. */\n readonly signal?: AbortSignal | undefined;\n /** False stops the client: it does not send the command again. A check that must fail at once uses it. */\n readonly reissue?: boolean | undefined;\n}\n\ntype Attempt =\n | { readonly outcome: \"ok\"; readonly value: unknown }\n | { readonly outcome: \"error\"; readonly error: string; readonly reply: boolean }\n | { readonly outcome: \"indeterminate\"; readonly error: string };\n\n/** The steps of a connection that the client gives a check for (B14). */\nexport type ClientStep = Extract<Step, \"dial\" | \"round-trip\" | \"local-state\">;\n\n/** Check options tell which checks the client gives the health runtime, and which policy each one controls. */\nexport interface CheckOptions {\n readonly poll: Poll;\n /** For each step, the policy that its check controls, or null for a check that only reports. A step that is absent has no check. */\n readonly steps: Partial<Record<ClientStep, Gate | null>>;\n}\n\n/** A Redis client is a dependency of a service: its declaration, its handle, its checks and its commands. */\nexport interface RedisClient {\n readonly name: string;\n /** This declaration goes to `svc.dependency(name, declaration, handle)`. */\n readonly declaration: Omit<Dependency, \"name\">;\n readonly handle: DependencyHandle;\n /** These are the manifestations that the client can meet, and the detector of each (E9). */\n readonly meets: readonly Meeting[];\n /** This text is the state of the connection, as ioredis names it, or \"none\" before the connect. */\n readonly status: string;\n /** These are the generations of the connection, the latest last. */\n readonly joins: readonly Join[];\n /**\n * This method sends a command, and gives its outcome in three values. The client sends again\n * only an idempotent operation, one time at most, after an outcome that is not certain. The\n * second try stays in the bound of the request. The client does not send again an operation that\n * the table does not name.\n */\n send<T = unknown>(command: string, args?: ReadonlyArray<string | number>, options?: SendOptions): Promise<Sent<T>>;\n /** This method gives the checks of the client and their functions, for `svc.check(check, run)`. */\n checks(options: CheckOptions): ReadonlyArray<readonly [Check, CheckFn]>;\n /** This method declares the client in a service: its dependency, its handle and its checks. */\n attach(svc: ServiceBuilder, options: CheckOptions): ServiceBuilder;\n /** This method gives the problems of the client in a description: a close that does not fit in its phase, for example. */\n problems(description: ServiceDescription): string[];\n}\n\nconst ms = (amount: Extent): number =>\n amount.value === \"infinite\" ? Number.POSITIVE_INFINITY : valueIn(amount as Quantity, \"ms\");\nconst finite = (amount: Extent): number | undefined => {\n const value = ms(amount);\n return Number.isFinite(value) ? value : undefined;\n};\nconst message = (error: unknown): string => (error instanceof Error ? error.message : String(error));\n\n/**\n * This function gives the time that the watchdog takes to find a silent server, at the most. A PING\n * starts each interval, from the start of the one before. Each one fails at its timeout, and the\n * last of `misses` failures in sequence closes the connection: `misses` intervals plus a timeout.\n */\nexport const heartbeatDetection = (interval: Extent, timeout: Quantity, misses: number): Extent =>\n interval.value === \"infinite\"\n ? { value: \"infinite\", unit: \"s\" }\n : quantity((misses * ms(interval) + ms(timeout)) / 1000, \"s\");\n\n/** This function gives the problems of a spec: the knobs that the client cannot use. */\nexport const specProblems = (name: string, spec: RedisClientSpec): string[] => {\n const problems: string[] = [];\n const { knobs } = spec;\n if (!Number.isInteger(spec.heartbeatMisses) || spec.heartbeatMisses < 1)\n problems.push(`${name}'s watchdog closes the connection after ${spec.heartbeatMisses} misses, not a whole number above zero`);\n if (knobs.max_connection_age.value !== \"infinite\")\n problems.push(`${name} cannot replace its connection at an age: max_connection_age must be \"infinite\"`);\n for (const [knob, amount] of Object.entries(knobs) as Array<[string, Extent]>) {\n if (amount.value !== \"infinite\" && !(ms(amount) >= 0))\n problems.push(`${name}'s ${knob} is not a time of zero or more`);\n }\n if (ms(knobs.request_timeout) <= 0) problems.push(`${name}'s request_timeout is not above zero: each command must have a bound`);\n if (ms(knobs.heartbeat_timeout) <= 0) problems.push(`${name}'s heartbeat_timeout is not above zero`);\n if (ms(knobs.retry_cap) < ms(knobs.retry_base)) problems.push(`${name}'s retry_cap is below its retry_base`);\n return problems;\n};\n\n/** This dial function opens a TCP connection, sends an inline PING, and waits for +PONG. */\nexport const tcpDial: DialFn = (host, port, signal) =>\n new Promise((resolve, reject) => {\n if (signal.aborted) {\n reject(new Error(\"the dial was aborted\"));\n return;\n }\n const socket = dialTcp({ host, port });\n let reply = \"\";\n const end = (error?: Error): void => {\n signal.removeEventListener(\"abort\", onAbort);\n socket.destroy();\n if (error === undefined) resolve();\n else reject(error);\n };\n const onAbort = (): void => end(new Error(\"the dial was aborted\"));\n signal.addEventListener(\"abort\", onAbort, { once: true });\n socket.once(\"connect\", () => socket.write(\"PING\\r\\n\"));\n socket.on(\"data\", (chunk) => {\n reply += chunk.toString();\n if (reply.includes(\"\\r\\n\")) end(reply.startsWith(\"+PONG\") ? undefined : new Error(`the reply was ${reply.trim()}`));\n });\n socket.once(\"error\", (error) => end(error));\n socket.once(\"close\", () => end(new Error(\"the server closed the dial\")));\n });\n\n/**\n * This function makes a Redis client. It is a dependency of a service, with ioredis under it:\n * - It sets each timer of ioredis from the declaration, and it does not let ioredis send a command\n * again (R5 §8).\n * - It tells the outcome of a command in three values, and it settles each command in flight as\n * indeterminate when its connection closes (C8, C10).\n * - It records the server's id of each connection at `ready` (D1).\n * - A watchdog sends a PING in the band of the connection, and it connects again after the misses\n * of the declaration. It also starts a client that stopped (principle 19).\n */\nexport const redisClient = (name: string, spec: RedisClientSpec, options: RedisClientOptions = {}): RedisClient => {\n const problems = specProblems(name, spec);\n if (problems.length > 0) throw new Error(problems.join(\"; \"));\n const { knobs } = spec;\n const clock = options.clock ?? realClock;\n const random = options.random ?? Math.random;\n const create = options.create ?? ioredis;\n const dial = options.dial ?? tcpDial;\n const turn = options.turn ?? (() => new Promise<void>((resolve) => setImmediate(resolve)));\n const operations: Readonly<Record<string, Operation>> = { ...REDIS_OPERATIONS, ...spec.operations };\n const requestTimeout = ms(knobs.request_timeout);\n const detection = heartbeatDetection(knobs.heartbeat_interval, knobs.heartbeat_timeout, spec.heartbeatMisses);\n\n let connection: RedisConnection | null = null;\n let status = \"none\";\n let closing = false;\n const joins: Join[] = [];\n // the commands that the client wrote while the connection was ready: a close settles each one\n const inFlight = new Set<(attempt: Attempt) => void>();\n const readyWaiters = new Set<() => void>();\n const listeners: Array<[string, (...args: unknown[]) => void]> = [];\n\n const record = (from: string, to: string, on: string, detail?: string): void =>\n options.record?.({ at: clock.now(), machine: \"client\", instance: name, from, to, on, ...(detail === undefined ? {} : { detail }) });\n\n // the delays of the connection attempts: doubling from the base to the cap, each with a jitter up\n // to the base, and none after the retry window. The watchdog starts the client again after that\n let retrySince: number | null = null;\n const retryStrategy = (times: number): number | null => {\n const now = clock.now();\n retrySince ??= now;\n if (now - retrySince > ms(knobs.client_retry_window)) {\n record(status, status, \"retry.window-ended\");\n return null;\n }\n const delay = Math.min(ms(knobs.retry_base) * 2 ** (times - 1), ms(knobs.retry_cap));\n return Math.round(delay + random() * ms(knobs.retry_base));\n };\n\n const connectionOptions: RedisConnectionOptions = {\n host: spec.host,\n port: spec.port,\n ...(spec.db === undefined ? {} : { db: spec.db }),\n ...(spec.username === undefined ? {} : { username: spec.username }),\n ...(spec.password === undefined ? {} : { password: spec.password }),\n lazyConnect: true,\n connectTimeout: ms(knobs.connect_bound),\n commandTimeout: requestTimeout,\n socketTimeout: finite(knobs.driver_writeoff_time),\n keepAlive: finite(knobs.keepalive_idle) ?? false,\n enableOfflineQueue: false,\n autoResendUnfulfilledCommands: false,\n maxRetriesPerRequest: null,\n enableReadyCheck: true,\n connectionName: name,\n retryStrategy,\n };\n\n /** This function gives the outcome of a failed command: a reply of the server, a command that did not leave the client, or neither. */\n const failed = (error: unknown): Attempt => {\n const text = message(error);\n if (error instanceof Error && error.name === \"ReplyError\") return { outcome: \"error\", error: text, reply: true };\n if (/isn't writeable|enableOfflineQueue/i.test(text)) return { outcome: \"error\", error: text, reply: false };\n return { outcome: \"indeterminate\", error: text };\n };\n\n const onReady = async (): Promise<void> => {\n retrySince = null;\n const c = connection;\n if (c === null) return;\n const generation = joins.length + 1;\n const at = clock.now();\n const socket = c.stream;\n let clientId: number | null = null;\n try {\n const reply = await c.call(\"CLIENT\", \"ID\");\n clientId = typeof reply === \"number\" ? reply : Number(reply);\n if (!Number.isFinite(clientId)) clientId = null;\n } catch {\n clientId = null;\n }\n // a connection that closed before its reply is a generation that never served\n if (connection !== c || c.status !== \"ready\") return;\n joins.push({\n generation,\n at,\n clientId,\n localAddress: socket?.localAddress ?? null,\n localPort: socket?.localPort ?? null,\n remoteAddress: socket?.remoteAddress ?? null,\n remotePort: socket?.remotePort ?? null,\n });\n record(\"ready\", \"ready\", \"join\", `generation ${generation}, client id ${clientId ?? \"unknown\"}`);\n for (const wake of [...readyWaiters]) wake();\n };\n\n const onStatus = (to: string) => (): void => {\n const from = status;\n status = to;\n record(from, to, `status.${to}`);\n if (to === \"close\" || to === \"end\") {\n // the lower layer gave up the connection: each command in flight learns it now (C10)\n for (const settle of [...inFlight]) settle({ outcome: \"indeterminate\", error: `the connection closed with the command in flight (${from})` });\n }\n if (to === \"ready\") void onReady();\n };\n\n const watch = (c: RedisConnection): void => {\n for (const state of [\"connecting\", \"connect\", \"ready\", \"close\", \"reconnecting\", \"end\"]) {\n const listener = onStatus(state);\n c.on(state, listener);\n listeners.push([state, listener]);\n }\n // an error of the connection comes before its close, which settles the commands: here it is only recorded\n const onError = (error: unknown): void => record(status, status, \"error\", message(error));\n c.on(\"error\", onError);\n listeners.push([\"error\", onError]);\n };\n\n /** This function waits until the connection is ready, for `within` milliseconds at most. It gives true when it is ready. */\n const waitReady = (within: number, signal?: AbortSignal): Promise<boolean> =>\n new Promise((resolve) => {\n if (connection?.status === \"ready\") {\n resolve(true);\n return;\n }\n let cancel: Cancel = () => {};\n const done = (value: boolean): void => {\n readyWaiters.delete(wake);\n cancel();\n signal?.removeEventListener(\"abort\", aborted);\n resolve(value);\n };\n const wake = (): void => done(true);\n const aborted = (): void => done(false);\n readyWaiters.add(wake);\n cancel = clock.after(within, () => done(connection?.status === \"ready\"));\n signal?.addEventListener(\"abort\", aborted, { once: true });\n });\n\n const attempt = (command: string, args: ReadonlyArray<string | number>, signal?: AbortSignal): Promise<Attempt> => {\n const c = connection;\n if (c === null) return Promise.resolve({ outcome: \"error\", error: `${name} has not connected`, reply: false });\n if (c.status !== \"ready\") return Promise.resolve({ outcome: \"error\", error: `${name} is ${c.status}, not ready: the command did not leave the client`, reply: false });\n if (signal?.aborted) return Promise.resolve({ outcome: \"error\", error: \"the caller aborted the command before it left the client\", reply: false });\n return new Promise<Attempt>((resolve) => {\n let settled = false;\n const settle = (value: Attempt): void => {\n if (settled) return;\n settled = true;\n inFlight.delete(settle);\n signal?.removeEventListener(\"abort\", aborted);\n resolve(value);\n };\n // a sent command cannot be cancelled: the caller abandons it, and its outcome is not known (principle 9)\n const aborted = (): void => settle({ outcome: \"indeterminate\", error: \"the caller abandoned the command after it was sent\" });\n inFlight.add(settle);\n signal?.addEventListener(\"abort\", aborted, { once: true });\n c.call(command, ...args).then(\n (value) => settle({ outcome: \"ok\", value }),\n (error: unknown) => settle(failed(error)),\n );\n });\n };\n\n const send = async <T>(command: string, args: ReadonlyArray<string | number> = [], sendOptions: SendOptions = {}): Promise<Sent<T>> => {\n const { signal } = sendOptions;\n const operation = sendOptions.reissue === false ? undefined : operations[commandName(command, args)];\n const started = clock.now();\n const first = await attempt(command, args, signal);\n if (first.outcome === \"ok\") return { outcome: \"ok\", value: first.value as T, attempts: 1 };\n // a reply of the server is final, and an operation that the table does not name is never sent again\n const again =\n !(first.outcome === \"error\" && first.reply) &&\n operation !== undefined &&\n signal?.aborted !== true &&\n mayReissue(operation, quantity((clock.now() - started) / 1000, \"s\"));\n const left = requestTimeout - (clock.now() - started);\n if (!again || left <= 0 || !(await waitReady(left, signal))) return { ...first, attempts: 1 };\n record(status, status, \"command.reissue\", commandName(command, args));\n // the second attempt ends at the bound of the request, not one command timeout after it starts\n const bound = new AbortController();\n const end = clock.after(Math.max(0, requestTimeout - (clock.now() - started)), () => bound.abort());\n const forward = (): void => bound.abort();\n signal?.addEventListener(\"abort\", forward, { once: true });\n const second = await attempt(command, args, bound.signal);\n end();\n signal?.removeEventListener(\"abort\", forward);\n return second.outcome === \"ok\" ? { outcome: \"ok\", value: second.value as T, attempts: 2 } : { ...second, attempts: 2 };\n };\n\n // the watchdog: a PING each interval from the start of the one before, on the connection of the\n // application. A reply, an error reply too, is an answer. It waits one turn of the event loop\n // before it counts a miss, thus a slow loop is not a silent server. It acts only on a ready\n // connection, thus it never closes a connection in the middle of a new attempt\n let misses = 0;\n let nextPing: Cancel | null = null;\n const heartbeatInterval = ms(knobs.heartbeat_interval);\n const ping = (): void => {\n nextPing = clock.after(heartbeatInterval, ping);\n const c = connection;\n if (c === null || closing) return;\n if (c.status === \"end\") {\n // the retry window ended: the watchdog starts the client again\n record(status, status, \"watchdog.revive\");\n retrySince = null;\n c.connect().catch(() => {});\n return;\n }\n if (c.status !== \"ready\") return;\n const controller = new AbortController();\n const timer = clock.after(ms(knobs.heartbeat_timeout), () => controller.abort());\n void attempt(\"PING\", [], controller.signal).then(async (result) => {\n timer();\n const answered = result.outcome === \"ok\" || (result.outcome === \"error\" && result.reply);\n await turn();\n if (answered) {\n misses = 0;\n return;\n }\n if (connection?.status !== \"ready\" || closing) return;\n misses += 1;\n record(status, status, \"watchdog.miss\", `${misses} of ${spec.heartbeatMisses}`);\n if (misses >= spec.heartbeatMisses) {\n misses = 0;\n record(status, status, \"watchdog.reconnect\");\n connection.disconnect(true);\n }\n });\n };\n\n const handle: DependencyHandle = {\n async connect(signal) {\n closing = false;\n const c = await create(connectionOptions);\n connection = c;\n watch(c);\n if (signal.aborted) throw new Error(`${name}: the service stopped before the connect`);\n // a server that is down at the start does not stop the startup: the client connects again in\n // the background, and its checks keep readiness tripped until it is ready (readiness is earned)\n await c.connect().catch((error: unknown) => record(status, status, \"connect.failed\", message(error)));\n if (Number.isFinite(heartbeatInterval)) nextPing = clock.after(heartbeatInterval, ping);\n },\n async close() {\n closing = true;\n nextPing?.();\n nextPing = null;\n const c = connection;\n if (c === null) return;\n // a QUIT lets the replies in flight arrive. A silent server never answers it, thus the close has its bound\n const quit = c.quit().then(\n () => \"quit\" as const,\n () => \"quit\" as const,\n );\n let cancel: Cancel = () => {};\n const bound = new Promise<\"bound\">((resolve) => {\n cancel = clock.after(ms(knobs.close_bound), () => resolve(\"bound\"));\n });\n const startedClose = clock.now();\n const first = await Promise.race([quit, bound]);\n cancel();\n if (first === \"bound\") record(status, status, \"close.bound\");\n // a client that is not ready refuses the QUIT at once, and it would connect again: the close stops it\n if (c.status !== \"end\") c.disconnect(false);\n // ioredis is at `end` when its socket closes, a moment after the reply: the close waits for it, in its bound\n const ended = (within: number): Promise<void> =>\n c.status === \"end\"\n ? Promise.resolve()\n : new Promise<void>((resolve) => {\n let stop: Cancel = () => {};\n const end = (): void => {\n c.off(\"end\", end);\n stop();\n resolve();\n };\n c.on(\"end\", end);\n stop = clock.after(Math.max(0, within), end);\n });\n await ended(ms(knobs.close_bound) - (clock.now() - startedClose));\n // a disconnect only half-closes the socket, and a silent server never closes its half: the bound destroys it\n if (c.status !== \"end\") {\n record(status, status, \"close.destroy\");\n c.stream?.destroy?.();\n await ended(0);\n }\n for (const [event, listener] of listeners) c.off(event, listener);\n listeners.length = 0;\n for (const settle of [...inFlight]) settle({ outcome: \"indeterminate\", error: \"the client closed with the command in flight\" });\n status = \"end\";\n },\n };\n\n const declaration: Omit<Dependency, \"name\"> = {\n system: spec.system,\n knobs: {\n connect_bound: knobs.connect_bound,\n request_timeout: knobs.request_timeout,\n driver_writeoff_time: knobs.driver_writeoff_time,\n keepalive_idle: knobs.keepalive_idle,\n heartbeat_interval: knobs.heartbeat_interval,\n heartbeat_timeout: knobs.heartbeat_timeout,\n heartbeat_detection_time: detection,\n client_retry_window: knobs.client_retry_window,\n retry_base: knobs.retry_base,\n retry_cap: knobs.retry_cap,\n close_bound: knobs.close_bound,\n max_connection_age: knobs.max_connection_age,\n retrans_giveup_time: knobs.retrans_giveup_time,\n },\n operations,\n };\n\n const checks = (checkOptions: CheckOptions): ReadonlyArray<readonly [Check, CheckFn]> => {\n const made: Array<readonly [Check, CheckFn]> = [];\n const declare = (step: ClientStep, ownConnection: boolean, run: CheckFn): void => {\n if (!(step in checkOptions.steps)) return;\n made.push([\n { name: `${name}.${step}`, dependency: name, step, ownConnection, gates: checkOptions.steps[step] ?? null, poll: checkOptions.poll },\n run,\n ]);\n };\n // the state of the client, without a round trip\n declare(\"local-state\", true, () => (connection?.status === \"ready\" ? \"ok\" : \"error\"));\n // a PING on the connection of the application: blind to nothing that the application meets\n declare(\"round-trip\", true, async (signal): Promise<Outcome> => {\n // a check measures the step now: it does not wait for a new connection, and it sends nothing again\n const result = await send(\"PING\", [], { signal, reissue: false });\n return result.outcome === \"ok\" ? \"ok\" : \"error\";\n });\n // a new connection each time: it finds a server that refuses connections, but not a stalled connection of the application\n declare(\"dial\", false, async (signal): Promise<Outcome> => {\n await dial(spec.host, spec.port, signal);\n return \"ok\";\n });\n return made;\n };\n\n return {\n name,\n declaration,\n handle,\n meets: meetings(name, knobs, detection),\n get status() {\n return connection === null ? \"none\" : connection.status;\n },\n joins,\n send,\n checks,\n attach(svc, checkOptions) {\n svc.dependency(name, declaration, handle);\n for (const [check, run] of checks(checkOptions)) svc.check(check, run);\n return svc;\n },\n problems(description) {\n const found: string[] = [];\n const teardown = description.shutdown.phases.find((phase) => phase.name === \"tear down dependencies\");\n if (teardown === undefined) found.push(`${name} needs the phase \"tear down dependencies\" to close`);\n else if (ms(knobs.close_bound) >= valueIn(teardown.cap, \"ms\"))\n found.push(`${name}'s close_bound of ${ms(knobs.close_bound)} ms does not fit in the cap of \"tear down dependencies\"`);\n for (const check of description.checks) {\n if (check.dependency !== name) continue;\n const slowest = valueIn(check.poll.timeout, \"ms\");\n if (check.step === \"round-trip\" && requestTimeout > slowest + valueIn(check.poll.interval, \"ms\") * (1 + check.poll.jitter))\n found.push(`${name}'s request_timeout outlives one jittered poll of ${check.name} (B7, O13)`);\n }\n return found;\n },\n };\n};\n","import type { Machine, TransitionDef } from \"@mark1russell7/algebra\";\n\nconst t = (id: string, from: string | readonly string[], on: string | readonly string[], to: string, extra: Partial<TransitionDef> = {}): TransitionDef => ({\n id,\n from,\n on,\n to,\n ...extra,\n});\nconst knob = (key: string) => ({ key, scope: \"client\", unit: \"s\", default: null }) as const;\n\n/** The states where the connection exists or tries to exist, thus where it can close. */\nconst LIVE = [\"connecting\", \"connect\", \"ready\", \"reconnecting\"] as const;\nconst ANY = [\"none\", \"connecting\", \"connect\", \"ready\", \"close\", \"reconnecting\", \"end\"] as const;\n\n/**\n * The Redis client as a `mark1russell7.machine/1` document. Its states are the states of ioredis.\n * The wrapper adds its own events: the join at each `ready`, and the misses of its watchdog. It also\n * adds a command that it sends again, and the bound of its close. A test records the transitions of the\n * client and replays them against this document.\n *\n * The state `ready` hides states that the client cannot see (D4). In a black hole, ioredis stays\n * `ready` while the kernel retransmits, until `driver_writeoff_time` or the watchdog ends that\n * hidden interval. A timer of this document bounds each hidden interval.\n */\nexport const REDIS_CLIENT_MACHINE: Machine = {\n format: \"mark1russell7.machine/1\",\n id: \"service.machine.redis-client\",\n title: \"The Redis client of the service, over ioredis 6\",\n layer: \"driver\",\n version: \"0.2.0\",\n sources: [{ ref: \"packages/service/src/redis (the client implements this document)\", rung: \"implemented\" }],\n states: {\n none: { initial: true, note: \"the client exists, and its connection does not\" },\n connecting: { note: \"a TCP connection attempt\" },\n connect: { note: \"the handshake: HELLO, then the ready check (INFO)\" },\n ready: { note: \"it can hide a black hole or a frozen server, until a timer of the client ends it\" },\n close: { note: \"the socket closed. Each command in flight settles as indeterminate\" },\n reconnecting: { note: \"the delay of the retry strategy, which doubles to its cap\" },\n end: { note: \"the retry window ended, or the service closed the client. The watchdog starts it again\" },\n },\n events: {\n \"status.connecting\": { kind: \"local\" },\n \"status.connect\": { kind: \"rx\" },\n \"status.ready\": { kind: \"rx\" },\n \"status.close\": { kind: \"rx\" },\n \"status.reconnecting\": { kind: \"timer\" },\n \"status.end\": { kind: \"local\" },\n join: { kind: \"rx\", note: \"the reply of CLIENT ID at ready (D1)\" },\n error: { kind: \"rx\" },\n \"connect.failed\": { kind: \"local\" },\n \"command.written\": { kind: \"tx\" },\n \"command.reissue\": { kind: \"local\" },\n data: { kind: \"rx\" },\n heartbeat: { kind: \"timer\" },\n \"watchdog.miss\": { kind: \"local\" },\n \"watchdog.reconnect\": { kind: \"local\" },\n \"watchdog.revive\": { kind: \"local\" },\n \"retry.window-ended\": { kind: \"local\" },\n \"close.request\": { kind: \"local\" },\n \"close.bound\": { kind: \"timer\" },\n \"close.destroy\": { kind: \"local\", note: \"the bound destroys a socket that a silent server does not close\" },\n },\n knobs: {\n connect_bound: knob(\"connectTimeout\"),\n driver_writeoff_time: knob(\"socketTimeout\"),\n heartbeat_interval: knob(\"watchdog.interval\"),\n close_bound: knob(\"close.bound\"),\n },\n timers: {\n connect_timeout: {\n duration: { knob: \"connect_bound\", unit: \"s\" },\n arms: [\"enter:connecting\"],\n cancels: [\"status.connect\", \"status.close\"],\n fires: \"status.close\",\n note: \"ioredis destroys the attempt\",\n },\n writeoff: {\n duration: { knob: \"driver_writeoff_time\", unit: \"s\" },\n arms: [\"command.written\"],\n cancels: [\"data\", \"status.close\"],\n fires: \"status.close\",\n note: \"ioredis destroys a socket that gives no data while a command waits: it ends the hidden interval of a black hole\",\n },\n heartbeat: {\n duration: { knob: \"heartbeat_interval\", unit: \"s\" },\n arms: [\"enter:ready\", \"heartbeat\"],\n cancels: [\"status.close\", \"status.end\"],\n fires: \"heartbeat\",\n note: \"a PING on the connection of the application\",\n },\n close: {\n duration: { knob: \"close_bound\", unit: \"s\" },\n arms: [\"close.request\"],\n cancels: [\"status.end\"],\n fires: \"close.bound\",\n },\n },\n transitions: [\n t(\"connect\", [\"none\", \"reconnecting\", \"end\", \"close\"], \"status.connecting\", \"connecting\"),\n t(\"tcp\", \"connecting\", \"status.connect\", \"connect\"),\n t(\"ready\", \"connect\", \"status.ready\", \"ready\"),\n t(\"close\", [...LIVE], \"status.close\", \"close\"),\n t(\"retry\", \"close\", \"status.reconnecting\", \"reconnecting\"),\n t(\"end\", [...LIVE, \"close\"], \"status.end\", \"end\"),\n t(\"join\", \"ready\", \"join\", \"ready\"),\n t(\"miss\", \"ready\", \"watchdog.miss\", \"ready\"),\n t(\"watchdog-reconnect\", \"ready\", \"watchdog.reconnect\", \"ready\"),\n t(\"revive\", \"end\", \"watchdog.revive\", \"end\"),\n // the retry strategy answers in `close`, before ioredis moves to `reconnecting` or `end`\n t(\"window-ended\", \"close\", \"retry.window-ended\", \"close\"),\n ...ANY.map((state) => t(`error-${state}`, state, \"error\", state)),\n ...ANY.map((state) => t(`connect-failed-${state}`, state, \"connect.failed\", state)),\n ...ANY.map((state) => t(`reissue-${state}`, state, \"command.reissue\", state)),\n ...ANY.map((state) => t(`close-bound-${state}`, state, \"close.bound\", state, { timer: \"close\" })),\n ...ANY.map((state) => t(`close-destroy-${state}`, state, \"close.destroy\", state)),\n ],\n};\n"],"mappings":";;;;;;;;AAkEA,MAAa,UAA6B,OAAO,YAAY;CAI3D,OAAO,KAAI,OAHW,OAAO,YAAA,CAGX,QAAQ,OAAO;AACnC;;;ACzCA,MAAM,QAAgB;CAAE,OAAO;CAAY,MAAM;AAAI;AACrD,MAAM,KAAK,UAA4B,SAAS,KAAK,MAAM,QAAQ,GAAG,IAAI,KAAK,GAAG;AAClF,MAAM,WAAW,WACf,OAAO,UAAU,aAAa,OAAO,oBAAoB,QAAQ,QAAoB,GAAG;AAC1F,MAAM,QAAQ,QAAgB,SAC5B,OAAO,UAAU,aAAa,QAAQ,EAAE,QAAQ,MAAM,IAAI,IAAI;;;;;;;AAYhE,MAAa,YAAY,MAAc,OAAmB,cAAiC;CACzF,MAAM,YAAY,KAAK,MAAM,gBAAgB,EAA2B;CACxE,MAAM,YAAY,eAAoC;EACpD,IAAI;EACJ,QAAQ;EACR;EACA,QAAQ,mBAAmB,KAAK;CAClC;CACA,MAAM,WAAqB;EACzB,IAAI;EACJ,QAAQ,MAAM;EACd,WAAW;EACX,QAAQ;CACV;CACA,MAAM,UAAoB;EACxB,IAAI;EACJ,QAAQ,MAAM;EACd,WAAW;EACX,QAAQ;CACV;CACA,MAAM,SAAmB;EACvB,IAAI;EACJ,QAAQ,MAAM;EACd,WAAW;EACX,QAAQ;CACV;CACA,MAAM,UAAU,eAAuB,UAA2B;EAChE;EACA;EACA,WAAW;GACT;GACA;GACA,SAAS,WAAW;GACpB;GACA,SAAS,MAAM;GACf;IAAE,IAAI;IAAkB,QAAQ;IAAW,WAAW;IAAQ,QAAQ;GAAgD;EACxH;CACF;CACA,MAAM,gBAAgB,eAAuB,UAA2B;EACtE;EACA;EACA,WAAW;GACT;GACA;GACA,SAAS,WAAW;GACpB,SAAS,MAAM;GACf;IAAE,IAAI;IAAkB,QAAQ;IAAO,WAAW;IAAQ,QAAQ;GAA8C;EAClH;CACF;CACA,MAAM,aAAa,WAAsB,YAA8B;EAAE,IAAI;EAAoB,QAAQ,EAAE,CAAC;EAAG;EAAW;CAAO;CACjI,OAAO;EACL;GACE,eAAe;GACf,MAAM;GACN,WAAW;IACT,UAAU,aAAa,+EAA+E;IACtG,SAAS,MAAM;IACf;KAAE,IAAI;KAAkB,QAAQ,UAAU,UAAU,aAAa,QAAQ,MAAM;KAAgB,WAAW;KAAQ,QAAQ;IAA+B;GAC3J;EACF;EACA;GACE,eAAe;GACf,MAAM;GACN,WAAW,CAAC,UAAU,aAAa,0CAA0C,GAAG,UAAU,QAAQ,mBAAmB,CAAC;EACxH;EACA,OAAO,aAAa,sDAAsD;EAC1E,OAAO,eAAe,0EAA0E;EAChG,aAAa,kBAAkB,0DAA0D;EACzF,aAAa,aAAa,8FAA8F;CAC1H;AACF;;AAGA,MAAa,iBAAiB,SAAkB,cAA0C;CAIxF,OAHe,QAAQ,UACpB,QAAQ,aAAa,SAAS,cAAc,aAAa,SAAS,OAAO,UAAU,UAAU,CAAC,CAC9F,MAAM,GAAG,MAAM,QAAQ,EAAE,MAAM,IAAI,QAAQ,EAAE,MAAM,CAC1C,CAAC,CAAC,MAAM;AACtB;;AAGA,MAAa,aAAa,UACxB,MAAM,SAAS,YACZ,CAAC,aAAa,MAAM,CAAC,CAAW,SAAS,cACxC,cAAc,SAAS,SAAS,MAAM,OAAO,CAAC;CAAE,eAAe,QAAQ;CAAe;AAAU,CAAC,IAAI,CAAC,CACxG,CACF;;;;;;;;;AC7HF,MAAM,cAAc;CAElB;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CAEA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;AACF;;;;;AAMA,MAAM,SAAS;CACb;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;AACF;;AAGA,MAAa,mBAAwD,OAAO,YAAY,CACtF,GAAG,YAAY,KAAK,YAAY,CAAC,SAAS,EAAE,YAAY,KAAK,CAAC,CAAU,GACxE,GAAG,OAAO,KAAK,YAAY,CAAC,SAAS,EAAE,YAAY,MAAM,CAAC,CAAU,CACtE,CAAC;;AAGD,MAAa,eAAe,SAAiB,SAAqC;CAChF,MAAM,OAAO,QAAQ,YAAY;CACjC,KAAK,SAAS,YAAY,SAAS,YAAY,SAAS,YAAY,OAAO,KAAK,OAAO,UACrF,OAAO,GAAG,KAAK,GAAG,KAAK,EAAE,CAAC,YAAY;CACxC,OAAO;AACT;;;ACgDA,MAAM,MAAM,WACV,OAAO,UAAU,aAAa,OAAO,oBAAoB,QAAQ,QAAoB,IAAI;AAC3F,MAAM,UAAU,WAAuC;CACrD,MAAM,QAAQ,GAAG,MAAM;CACvB,OAAO,OAAO,SAAS,KAAK,IAAI,QAAQ,KAAA;AAC1C;AACA,MAAM,WAAW,UAA4B,iBAAiB,QAAQ,MAAM,UAAU,OAAO,KAAK;;;;;;AAOlG,MAAa,sBAAsB,UAAkB,SAAmB,WACtE,SAAS,UAAU,aACf;CAAE,OAAO;CAAY,MAAM;AAAI,IAC/B,UAAU,SAAS,GAAG,QAAQ,IAAI,GAAG,OAAO,KAAK,KAAM,GAAG;;AAGhE,MAAa,gBAAgB,MAAc,SAAoC;CAC7E,MAAM,WAAqB,CAAC;CAC5B,MAAM,EAAE,UAAU;CAClB,IAAI,CAAC,OAAO,UAAU,KAAK,eAAe,KAAK,KAAK,kBAAkB,GACpE,SAAS,KAAK,GAAG,KAAK,0CAA0C,KAAK,gBAAgB,uCAAuC;CAC9H,IAAI,MAAM,mBAAmB,UAAU,YACrC,SAAS,KAAK,GAAG,KAAK,gFAAgF;CACxG,KAAK,MAAM,CAAC,MAAM,WAAW,OAAO,QAAQ,KAAK,GAC/C,IAAI,OAAO,UAAU,cAAc,EAAE,GAAG,MAAM,KAAK,IACjD,SAAS,KAAK,GAAG,KAAK,KAAK,KAAK,+BAA+B;CAEnE,IAAI,GAAG,MAAM,eAAe,KAAK,GAAG,SAAS,KAAK,GAAG,KAAK,qEAAqE;CAC/H,IAAI,GAAG,MAAM,iBAAiB,KAAK,GAAG,SAAS,KAAK,GAAG,KAAK,uCAAuC;CACnG,IAAI,GAAG,MAAM,SAAS,IAAI,GAAG,MAAM,UAAU,GAAG,SAAS,KAAK,GAAG,KAAK,qCAAqC;CAC3G,OAAO;AACT;;AAGA,MAAa,WAAmB,MAAM,MAAM,WAC1C,IAAI,SAAS,SAAS,WAAW;CAC/B,IAAI,OAAO,SAAS;EAClB,uBAAO,IAAI,MAAM,sBAAsB,CAAC;EACxC;CACF;CACA,MAAM,SAASA,QAAQ;EAAE;EAAM;CAAK,CAAC;CACrC,IAAI,QAAQ;CACZ,MAAM,OAAO,UAAwB;EACnC,OAAO,oBAAoB,SAAS,OAAO;EAC3C,OAAO,QAAQ;EACf,IAAI,UAAU,KAAA,GAAW,QAAQ;OAC5B,OAAO,KAAK;CACnB;CACA,MAAM,gBAAsB,oBAAI,IAAI,MAAM,sBAAsB,CAAC;CACjE,OAAO,iBAAiB,SAAS,SAAS,EAAE,MAAM,KAAK,CAAC;CACxD,OAAO,KAAK,iBAAiB,OAAO,MAAM,UAAU,CAAC;CACrD,OAAO,GAAG,SAAS,UAAU;EAC3B,SAAS,MAAM,SAAS;EACxB,IAAI,MAAM,SAAS,MAAM,GAAG,IAAI,MAAM,WAAW,OAAO,IAAI,KAAA,oBAAY,IAAI,MAAM,iBAAiB,MAAM,KAAK,GAAG,CAAC;CACpH,CAAC;CACD,OAAO,KAAK,UAAU,UAAU,IAAI,KAAK,CAAC;CAC1C,OAAO,KAAK,eAAe,oBAAI,IAAI,MAAM,4BAA4B,CAAC,CAAC;AACzE,CAAC;;;;;;;;;;;AAYH,MAAa,eAAe,MAAc,MAAuB,UAA8B,CAAC,MAAmB;CACjH,MAAM,WAAW,aAAa,MAAM,IAAI;CACxC,IAAI,SAAS,SAAS,GAAG,MAAM,IAAI,MAAM,SAAS,KAAK,IAAI,CAAC;CAC5D,MAAM,EAAE,UAAU;CAClB,MAAM,QAAQ,QAAQ,SAAS;CAC/B,MAAM,SAAS,QAAQ,UAAU,KAAK;CACtC,MAAM,SAAS,QAAQ,UAAU;CACjC,MAAM,OAAO,QAAQ,QAAQ;CAC7B,MAAM,OAAO,QAAQ,eAAe,IAAI,SAAe,YAAY,aAAa,OAAO,CAAC;CACxF,MAAM,aAAkD;EAAE,GAAG;EAAkB,GAAG,KAAK;CAAW;CAClG,MAAM,iBAAiB,GAAG,MAAM,eAAe;CAC/C,MAAM,YAAY,mBAAmB,MAAM,oBAAoB,MAAM,mBAAmB,KAAK,eAAe;CAE5G,IAAI,aAAqC;CACzC,IAAI,SAAS;CACb,IAAI,UAAU;CACd,MAAM,QAAgB,CAAC;CAEvB,MAAM,2BAAW,IAAI,IAAgC;CACrD,MAAM,+BAAe,IAAI,IAAgB;CACzC,MAAM,YAA2D,CAAC;CAElE,MAAM,UAAU,MAAc,IAAY,IAAY,WACpD,QAAQ,SAAS;EAAE,IAAI,MAAM,IAAI;EAAG,SAAS;EAAU,UAAU;EAAM;EAAM;EAAI;EAAI,GAAI,WAAW,KAAA,IAAY,CAAC,IAAI,EAAE,OAAO;CAAG,CAAC;CAIpI,IAAI,aAA4B;CAChC,MAAM,iBAAiB,UAAiC;EACtD,MAAM,MAAM,MAAM,IAAI;EACtB,eAAe;EACf,IAAI,MAAM,aAAa,GAAG,MAAM,mBAAmB,GAAG;GACpD,OAAO,QAAQ,QAAQ,oBAAoB;GAC3C,OAAO;EACT;EACA,MAAM,QAAQ,KAAK,IAAI,GAAG,MAAM,UAAU,IAAI,MAAM,QAAQ,IAAI,GAAG,MAAM,SAAS,CAAC;EACnF,OAAO,KAAK,MAAM,QAAQ,OAAO,IAAI,GAAG,MAAM,UAAU,CAAC;CAC3D;CAEA,MAAM,oBAA4C;EAChD,MAAM,KAAK;EACX,MAAM,KAAK;EACX,GAAI,KAAK,OAAO,KAAA,IAAY,CAAC,IAAI,EAAE,IAAI,KAAK,GAAG;EAC/C,GAAI,KAAK,aAAa,KAAA,IAAY,CAAC,IAAI,EAAE,UAAU,KAAK,SAAS;EACjE,GAAI,KAAK,aAAa,KAAA,IAAY,CAAC,IAAI,EAAE,UAAU,KAAK,SAAS;EACjE,aAAa;EACb,gBAAgB,GAAG,MAAM,aAAa;EACtC,gBAAgB;EAChB,eAAe,OAAO,MAAM,oBAAoB;EAChD,WAAW,OAAO,MAAM,cAAc,KAAK;EAC3C,oBAAoB;EACpB,+BAA+B;EAC/B,sBAAsB;EACtB,kBAAkB;EAClB,gBAAgB;EAChB;CACF;;CAGA,MAAM,UAAU,UAA4B;EAC1C,MAAM,OAAO,QAAQ,KAAK;EAC1B,IAAI,iBAAiB,SAAS,MAAM,SAAS,cAAc,OAAO;GAAE,SAAS;GAAS,OAAO;GAAM,OAAO;EAAK;EAC/G,IAAI,sCAAsC,KAAK,IAAI,GAAG,OAAO;GAAE,SAAS;GAAS,OAAO;GAAM,OAAO;EAAM;EAC3G,OAAO;GAAE,SAAS;GAAiB,OAAO;EAAK;CACjD;CAEA,MAAM,UAAU,YAA2B;EACzC,aAAa;EACb,MAAM,IAAI;EACV,IAAI,MAAM,MAAM;EAChB,MAAM,aAAa,MAAM,SAAS;EAClC,MAAM,KAAK,MAAM,IAAI;EACrB,MAAM,SAAS,EAAE;EACjB,IAAI,WAA0B;EAC9B,IAAI;GACF,MAAM,QAAQ,MAAM,EAAE,KAAK,UAAU,IAAI;GACzC,WAAW,OAAO,UAAU,WAAW,QAAQ,OAAO,KAAK;GAC3D,IAAI,CAAC,OAAO,SAAS,QAAQ,GAAG,WAAW;EAC7C,QAAQ;GACN,WAAW;EACb;EAEA,IAAI,eAAe,KAAK,EAAE,WAAW,SAAS;EAC9C,MAAM,KAAK;GACT;GACA;GACA;GACA,cAAc,QAAQ,gBAAgB;GACtC,WAAW,QAAQ,aAAa;GAChC,eAAe,QAAQ,iBAAiB;GACxC,YAAY,QAAQ,cAAc;EACpC,CAAC;EACD,OAAO,SAAS,SAAS,QAAQ,cAAc,WAAW,cAAc,YAAY,WAAW;EAC/F,KAAK,MAAM,QAAQ,CAAC,GAAG,YAAY,GAAG,KAAK;CAC7C;CAEA,MAAM,YAAY,aAA2B;EAC3C,MAAM,OAAO;EACb,SAAS;EACT,OAAO,MAAM,IAAI,UAAU,IAAI;EAC/B,IAAI,OAAO,WAAW,OAAO,OAE3B,KAAK,MAAM,UAAU,CAAC,GAAG,QAAQ,GAAG,OAAO;GAAE,SAAS;GAAiB,OAAO,qDAAqD,KAAK;EAAG,CAAC;EAE9I,IAAI,OAAO,SAAS,QAAa;CACnC;CAEA,MAAM,SAAS,MAA6B;EAC1C,KAAK,MAAM,SAAS;GAAC;GAAc;GAAW;GAAS;GAAS;GAAgB;EAAK,GAAG;GACtF,MAAM,WAAW,SAAS,KAAK;GAC/B,EAAE,GAAG,OAAO,QAAQ;GACpB,UAAU,KAAK,CAAC,OAAO,QAAQ,CAAC;EAClC;EAEA,MAAM,WAAW,UAAyB,OAAO,QAAQ,QAAQ,SAAS,QAAQ,KAAK,CAAC;EACxF,EAAE,GAAG,SAAS,OAAO;EACrB,UAAU,KAAK,CAAC,SAAS,OAAO,CAAC;CACnC;;CAGA,MAAM,aAAa,QAAgB,WACjC,IAAI,SAAS,YAAY;EACvB,IAAI,YAAY,WAAW,SAAS;GAClC,QAAQ,IAAI;GACZ;EACF;EACA,IAAI,eAAuB,CAAC;EAC5B,MAAM,QAAQ,UAAyB;GACrC,aAAa,OAAO,IAAI;GACxB,OAAO;GACP,QAAQ,oBAAoB,SAAS,OAAO;GAC5C,QAAQ,KAAK;EACf;EACA,MAAM,aAAmB,KAAK,IAAI;EAClC,MAAM,gBAAsB,KAAK,KAAK;EACtC,aAAa,IAAI,IAAI;EACrB,SAAS,MAAM,MAAM,cAAc,KAAK,YAAY,WAAW,OAAO,CAAC;EACvE,QAAQ,iBAAiB,SAAS,SAAS,EAAE,MAAM,KAAK,CAAC;CAC3D,CAAC;CAEH,MAAM,WAAW,SAAiB,MAAsC,WAA2C;EACjH,MAAM,IAAI;EACV,IAAI,MAAM,MAAM,OAAO,QAAQ,QAAQ;GAAE,SAAS;GAAS,OAAO,GAAG,KAAK;GAAqB,OAAO;EAAM,CAAC;EAC7G,IAAI,EAAE,WAAW,SAAS,OAAO,QAAQ,QAAQ;GAAE,SAAS;GAAS,OAAO,GAAG,KAAK,MAAM,EAAE,OAAO;GAAoD,OAAO;EAAM,CAAC;EACrK,IAAI,QAAQ,SAAS,OAAO,QAAQ,QAAQ;GAAE,SAAS;GAAS,OAAO;GAA4D,OAAO;EAAM,CAAC;EACjJ,OAAO,IAAI,SAAkB,YAAY;GACvC,IAAI,UAAU;GACd,MAAM,UAAU,UAAyB;IACvC,IAAI,SAAS;IACb,UAAU;IACV,SAAS,OAAO,MAAM;IACtB,QAAQ,oBAAoB,SAAS,OAAO;IAC5C,QAAQ,KAAK;GACf;GAEA,MAAM,gBAAsB,OAAO;IAAE,SAAS;IAAiB,OAAO;GAAqD,CAAC;GAC5H,SAAS,IAAI,MAAM;GACnB,QAAQ,iBAAiB,SAAS,SAAS,EAAE,MAAM,KAAK,CAAC;GACzD,EAAE,KAAK,SAAS,GAAG,IAAI,CAAC,CAAC,MACtB,UAAU,OAAO;IAAE,SAAS;IAAM;GAAM,CAAC,IACzC,UAAmB,OAAO,OAAO,KAAK,CAAC,CAC1C;EACF,CAAC;CACH;CAEA,MAAM,OAAO,OAAU,SAAiB,OAAuC,CAAC,GAAG,cAA2B,CAAC,MAAwB;EACrI,MAAM,EAAE,WAAW;EACnB,MAAM,YAAY,YAAY,YAAY,QAAQ,KAAA,IAAY,WAAW,YAAY,SAAS,IAAI;EAClG,MAAM,UAAU,MAAM,IAAI;EAC1B,MAAM,QAAQ,MAAM,QAAQ,SAAS,MAAM,MAAM;EACjD,IAAI,MAAM,YAAY,MAAM,OAAO;GAAE,SAAS;GAAM,OAAO,MAAM;GAAY,UAAU;EAAE;EAEzF,MAAM,QACJ,EAAE,MAAM,YAAY,WAAW,MAAM,UACrC,cAAc,KAAA,KACd,QAAQ,YAAY,QACpB,WAAW,WAAW,UAAU,MAAM,IAAI,IAAI,WAAW,KAAM,GAAG,CAAC;EACrE,MAAM,OAAO,kBAAkB,MAAM,IAAI,IAAI;EAC7C,IAAI,CAAC,SAAS,QAAQ,KAAK,CAAE,MAAM,UAAU,MAAM,MAAM,GAAI,OAAO;GAAE,GAAG;GAAO,UAAU;EAAE;EAC5F,OAAO,QAAQ,QAAQ,mBAAmB,YAAY,SAAS,IAAI,CAAC;EAEpE,MAAM,QAAQ,IAAI,gBAAgB;EAClC,MAAM,MAAM,MAAM,MAAM,KAAK,IAAI,GAAG,kBAAkB,MAAM,IAAI,IAAI,QAAQ,SAAS,MAAM,MAAM,CAAC;EAClG,MAAM,gBAAsB,MAAM,MAAM;EACxC,QAAQ,iBAAiB,SAAS,SAAS,EAAE,MAAM,KAAK,CAAC;EACzD,MAAM,SAAS,MAAM,QAAQ,SAAS,MAAM,MAAM,MAAM;EACxD,IAAI;EACJ,QAAQ,oBAAoB,SAAS,OAAO;EAC5C,OAAO,OAAO,YAAY,OAAO;GAAE,SAAS;GAAM,OAAO,OAAO;GAAY,UAAU;EAAE,IAAI;GAAE,GAAG;GAAQ,UAAU;EAAE;CACvH;CAMA,IAAI,SAAS;CACb,IAAI,WAA0B;CAC9B,MAAM,oBAAoB,GAAG,MAAM,kBAAkB;CACrD,MAAM,aAAmB;EACvB,WAAW,MAAM,MAAM,mBAAmB,IAAI;EAC9C,MAAM,IAAI;EACV,IAAI,MAAM,QAAQ,SAAS;EAC3B,IAAI,EAAE,WAAW,OAAO;GAEtB,OAAO,QAAQ,QAAQ,iBAAiB;GACxC,aAAa;GACb,EAAE,QAAQ,CAAC,CAAC,YAAY,CAAC,CAAC;GAC1B;EACF;EACA,IAAI,EAAE,WAAW,SAAS;EAC1B,MAAM,aAAa,IAAI,gBAAgB;EACvC,MAAM,QAAQ,MAAM,MAAM,GAAG,MAAM,iBAAiB,SAAS,WAAW,MAAM,CAAC;EAC/E,QAAa,QAAQ,CAAC,GAAG,WAAW,MAAM,CAAC,CAAC,KAAK,OAAO,WAAW;GACjE,MAAM;GACN,MAAM,WAAW,OAAO,YAAY,QAAS,OAAO,YAAY,WAAW,OAAO;GAClF,MAAM,KAAK;GACX,IAAI,UAAU;IACZ,SAAS;IACT;GACF;GACA,IAAI,YAAY,WAAW,WAAW,SAAS;GAC/C,UAAU;GACV,OAAO,QAAQ,QAAQ,iBAAiB,GAAG,OAAO,MAAM,KAAK,iBAAiB;GAC9E,IAAI,UAAU,KAAK,iBAAiB;IAClC,SAAS;IACT,OAAO,QAAQ,QAAQ,oBAAoB;IAC3C,WAAW,WAAW,IAAI;GAC5B;EACF,CAAC;CACH;CAEA,MAAM,SAA2B;EAC/B,MAAM,QAAQ,QAAQ;GACpB,UAAU;GACV,MAAM,IAAI,MAAM,OAAO,iBAAiB;GACxC,aAAa;GACb,MAAM,CAAC;GACP,IAAI,OAAO,SAAS,MAAM,IAAI,MAAM,GAAG,KAAK,yCAAyC;GAGrF,MAAM,EAAE,QAAQ,CAAC,CAAC,OAAO,UAAmB,OAAO,QAAQ,QAAQ,kBAAkB,QAAQ,KAAK,CAAC,CAAC;GACpG,IAAI,OAAO,SAAS,iBAAiB,GAAG,WAAW,MAAM,MAAM,mBAAmB,IAAI;EACxF;EACA,MAAM,QAAQ;GACZ,UAAU;GACV,WAAW;GACX,WAAW;GACX,MAAM,IAAI;GACV,IAAI,MAAM,MAAM;GAEhB,MAAM,OAAO,EAAE,KAAK,CAAC,CAAC,WACd,cACA,MACR;GACA,IAAI,eAAuB,CAAC;GAC5B,MAAM,QAAQ,IAAI,SAAkB,YAAY;IAC9C,SAAS,MAAM,MAAM,GAAG,MAAM,WAAW,SAAS,QAAQ,OAAO,CAAC;GACpE,CAAC;GACD,MAAM,eAAe,MAAM,IAAI;GAC/B,MAAM,QAAQ,MAAM,QAAQ,KAAK,CAAC,MAAM,KAAK,CAAC;GAC9C,OAAO;GACP,IAAI,UAAU,SAAS,OAAO,QAAQ,QAAQ,aAAa;GAE3D,IAAI,EAAE,WAAW,OAAO,EAAE,WAAW,KAAK;GAE1C,MAAM,SAAS,WACb,EAAE,WAAW,QACT,QAAQ,QAAQ,IAChB,IAAI,SAAe,YAAY;IAC7B,IAAI,aAAqB,CAAC;IAC1B,MAAM,YAAkB;KACtB,EAAE,IAAI,OAAO,GAAG;KAChB,KAAK;KACL,QAAQ;IACV;IACA,EAAE,GAAG,OAAO,GAAG;IACf,OAAO,MAAM,MAAM,KAAK,IAAI,GAAG,MAAM,GAAG,GAAG;GAC7C,CAAC;GACP,MAAM,MAAM,GAAG,MAAM,WAAW,KAAK,MAAM,IAAI,IAAI,aAAa;GAEhE,IAAI,EAAE,WAAW,OAAO;IACtB,OAAO,QAAQ,QAAQ,eAAe;IACtC,EAAE,QAAQ,UAAU;IACpB,MAAM,MAAM,CAAC;GACf;GACA,KAAK,MAAM,CAAC,OAAO,aAAa,WAAW,EAAE,IAAI,OAAO,QAAQ;GAChE,UAAU,SAAS;GACnB,KAAK,MAAM,UAAU,CAAC,GAAG,QAAQ,GAAG,OAAO;IAAE,SAAS;IAAiB,OAAO;GAA+C,CAAC;GAC9H,SAAS;EACX;CACF;CAEA,MAAM,cAAwC;EAC5C,QAAQ,KAAK;EACb,OAAO;GACL,eAAe,MAAM;GACrB,iBAAiB,MAAM;GACvB,sBAAsB,MAAM;GAC5B,gBAAgB,MAAM;GACtB,oBAAoB,MAAM;GAC1B,mBAAmB,MAAM;GACzB,0BAA0B;GAC1B,qBAAqB,MAAM;GAC3B,YAAY,MAAM;GAClB,WAAW,MAAM;GACjB,aAAa,MAAM;GACnB,oBAAoB,MAAM;GAC1B,qBAAqB,MAAM;EAC7B;EACA;CACF;CAEA,MAAM,UAAU,iBAAyE;EACvF,MAAM,OAAyC,CAAC;EAChD,MAAM,WAAW,MAAkB,eAAwB,QAAuB;GAChF,IAAI,EAAE,QAAQ,aAAa,QAAQ;GACnC,KAAK,KAAK,CACR;IAAE,MAAM,GAAG,KAAK,GAAG;IAAQ,YAAY;IAAM;IAAM;IAAe,OAAO,aAAa,MAAM,SAAS;IAAM,MAAM,aAAa;GAAK,GACnI,GACF,CAAC;EACH;EAEA,QAAQ,eAAe,YAAa,YAAY,WAAW,UAAU,OAAO,OAAQ;EAEpF,QAAQ,cAAc,MAAM,OAAO,WAA6B;GAG9D,QAAO,MADc,KAAK,QAAQ,CAAC,GAAG;IAAE;IAAQ,SAAS;GAAM,CAAC,EAAA,CAClD,YAAY,OAAO,OAAO;EAC1C,CAAC;EAED,QAAQ,QAAQ,OAAO,OAAO,WAA6B;GACzD,MAAM,KAAK,KAAK,MAAM,KAAK,MAAM,MAAM;GACvC,OAAO;EACT,CAAC;EACD,OAAO;CACT;CAEA,OAAO;EACL;EACA;EACA;EACA,OAAO,SAAS,MAAM,OAAO,SAAS;EACtC,IAAI,SAAS;GACX,OAAO,eAAe,OAAO,SAAS,WAAW;EACnD;EACA;EACA;EACA;EACA,OAAO,KAAK,cAAc;GACxB,IAAI,WAAW,MAAM,aAAa,MAAM;GACxC,KAAK,MAAM,CAAC,OAAO,QAAQ,OAAO,YAAY,GAAG,IAAI,MAAM,OAAO,GAAG;GACrE,OAAO;EACT;EACA,SAAS,aAAa;GACpB,MAAM,QAAkB,CAAC;GACzB,MAAM,WAAW,YAAY,SAAS,OAAO,MAAM,UAAU,MAAM,SAAS,wBAAwB;GACpG,IAAI,aAAa,KAAA,GAAW,MAAM,KAAK,GAAG,KAAK,mDAAmD;QAC7F,IAAI,GAAG,MAAM,WAAW,KAAK,QAAQ,SAAS,KAAK,IAAI,GAC1D,MAAM,KAAK,GAAG,KAAK,oBAAoB,GAAG,MAAM,WAAW,EAAE,wDAAwD;GACvH,KAAK,MAAM,SAAS,YAAY,QAAQ;IACtC,IAAI,MAAM,eAAe,MAAM;IAC/B,MAAM,UAAU,QAAQ,MAAM,KAAK,SAAS,IAAI;IAChD,IAAI,MAAM,SAAS,gBAAgB,iBAAiB,UAAU,QAAQ,MAAM,KAAK,UAAU,IAAI,KAAK,IAAI,MAAM,KAAK,SACjH,MAAM,KAAK,GAAG,KAAK,mDAAmD,MAAM,KAAK,WAAW;GAChG;GACA,OAAO;EACT;CACF;AACF;;;AC9lBA,MAAM,KAAK,IAAY,MAAkC,IAAgC,IAAY,QAAgC,CAAC,OAAsB;CAC1J;CACA;CACA;CACA;CACA,GAAG;AACL;AACA,MAAM,QAAQ,SAAiB;CAAE;CAAK,OAAO;CAAU,MAAM;CAAK,SAAS;AAAK;;AAGhF,MAAM,OAAO;CAAC;CAAc;CAAW;CAAS;AAAc;AAC9D,MAAM,MAAM;CAAC;CAAQ;CAAc;CAAW;CAAS;CAAS;CAAgB;AAAK;;;;;;;;;;;AAYrF,MAAa,uBAAgC;CAC3C,QAAQ;CACR,IAAI;CACJ,OAAO;CACP,OAAO;CACP,SAAS;CACT,SAAS,CAAC;EAAE,KAAK;EAAoE,MAAM;CAAc,CAAC;CAC1G,QAAQ;EACN,MAAM;GAAE,SAAS;GAAM,MAAM;EAAiD;EAC9E,YAAY,EAAE,MAAM,2BAA2B;EAC/C,SAAS,EAAE,MAAM,oDAAoD;EACrE,OAAO,EAAE,MAAM,mFAAmF;EAClG,OAAO,EAAE,MAAM,qEAAqE;EACpF,cAAc,EAAE,MAAM,4DAA4D;EAClF,KAAK,EAAE,MAAM,yFAAyF;CACxG;CACA,QAAQ;EACN,qBAAqB,EAAE,MAAM,QAAQ;EACrC,kBAAkB,EAAE,MAAM,KAAK;EAC/B,gBAAgB,EAAE,MAAM,KAAK;EAC7B,gBAAgB,EAAE,MAAM,KAAK;EAC7B,uBAAuB,EAAE,MAAM,QAAQ;EACvC,cAAc,EAAE,MAAM,QAAQ;EAC9B,MAAM;GAAE,MAAM;GAAM,MAAM;EAAuC;EACjE,OAAO,EAAE,MAAM,KAAK;EACpB,kBAAkB,EAAE,MAAM,QAAQ;EAClC,mBAAmB,EAAE,MAAM,KAAK;EAChC,mBAAmB,EAAE,MAAM,QAAQ;EACnC,MAAM,EAAE,MAAM,KAAK;EACnB,WAAW,EAAE,MAAM,QAAQ;EAC3B,iBAAiB,EAAE,MAAM,QAAQ;EACjC,sBAAsB,EAAE,MAAM,QAAQ;EACtC,mBAAmB,EAAE,MAAM,QAAQ;EACnC,sBAAsB,EAAE,MAAM,QAAQ;EACtC,iBAAiB,EAAE,MAAM,QAAQ;EACjC,eAAe,EAAE,MAAM,QAAQ;EAC/B,iBAAiB;GAAE,MAAM;GAAS,MAAM;EAAkE;CAC5G;CACA,OAAO;EACL,eAAe,KAAK,gBAAgB;EACpC,sBAAsB,KAAK,eAAe;EAC1C,oBAAoB,KAAK,mBAAmB;EAC5C,aAAa,KAAK,aAAa;CACjC;CACA,QAAQ;EACN,iBAAiB;GACf,UAAU;IAAE,MAAM;IAAiB,MAAM;GAAI;GAC7C,MAAM,CAAC,kBAAkB;GACzB,SAAS,CAAC,kBAAkB,cAAc;GAC1C,OAAO;GACP,MAAM;EACR;EACA,UAAU;GACR,UAAU;IAAE,MAAM;IAAwB,MAAM;GAAI;GACpD,MAAM,CAAC,iBAAiB;GACxB,SAAS,CAAC,QAAQ,cAAc;GAChC,OAAO;GACP,MAAM;EACR;EACA,WAAW;GACT,UAAU;IAAE,MAAM;IAAsB,MAAM;GAAI;GAClD,MAAM,CAAC,eAAe,WAAW;GACjC,SAAS,CAAC,gBAAgB,YAAY;GACtC,OAAO;GACP,MAAM;EACR;EACA,OAAO;GACL,UAAU;IAAE,MAAM;IAAe,MAAM;GAAI;GAC3C,MAAM,CAAC,eAAe;GACtB,SAAS,CAAC,YAAY;GACtB,OAAO;EACT;CACF;CACA,aAAa;EACX,EAAE,WAAW;GAAC;GAAQ;GAAgB;GAAO;EAAO,GAAG,qBAAqB,YAAY;EACxF,EAAE,OAAO,cAAc,kBAAkB,SAAS;EAClD,EAAE,SAAS,WAAW,gBAAgB,OAAO;EAC7C,EAAE,SAAS,CAAC,GAAG,IAAI,GAAG,gBAAgB,OAAO;EAC7C,EAAE,SAAS,SAAS,uBAAuB,cAAc;EACzD,EAAE,OAAO,CAAC,GAAG,MAAM,OAAO,GAAG,cAAc,KAAK;EAChD,EAAE,QAAQ,SAAS,QAAQ,OAAO;EAClC,EAAE,QAAQ,SAAS,iBAAiB,OAAO;EAC3C,EAAE,sBAAsB,SAAS,sBAAsB,OAAO;EAC9D,EAAE,UAAU,OAAO,mBAAmB,KAAK;EAE3C,EAAE,gBAAgB,SAAS,sBAAsB,OAAO;EACxD,GAAG,IAAI,KAAK,UAAU,EAAE,SAAS,SAAS,OAAO,SAAS,KAAK,CAAC;EAChE,GAAG,IAAI,KAAK,UAAU,EAAE,kBAAkB,SAAS,OAAO,kBAAkB,KAAK,CAAC;EAClF,GAAG,IAAI,KAAK,UAAU,EAAE,WAAW,SAAS,OAAO,mBAAmB,KAAK,CAAC;EAC5E,GAAG,IAAI,KAAK,UAAU,EAAE,eAAe,SAAS,OAAO,eAAe,OAAO,EAAE,OAAO,QAAQ,CAAC,CAAC;EAChG,GAAG,IAAI,KAAK,UAAU,EAAE,iBAAiB,SAAS,OAAO,iBAAiB,KAAK,CAAC;CAClF;AACF"}
|
|
@@ -0,0 +1,102 @@
|
|
|
1
|
+
import { C as ProcessPort, M as Outcome, R as Clock, S as ExitSignal, j as CheckFn, n as ServiceBuilder } from "./builder-BFxOuULH.js";
|
|
2
|
+
//#region src/fakes.d.ts
|
|
3
|
+
/** A fake clock moves only when a test tells it to. */
|
|
4
|
+
interface FakeClock extends Clock {
|
|
5
|
+
/** This method moves the clock forward by `ms`, and it starts each timer that comes due, in sequence. */
|
|
6
|
+
advance(ms: number): Promise<void>;
|
|
7
|
+
/**
|
|
8
|
+
* This method moves the clock forward by `ms` and starts no timer, as a blocked event loop does.
|
|
9
|
+
* The next `advance` then starts each timer that came due during the block, late.
|
|
10
|
+
*/
|
|
11
|
+
block(ms: number): void;
|
|
12
|
+
/** This count is the number of timers that are not stopped and not called. */
|
|
13
|
+
readonly pending: number;
|
|
14
|
+
}
|
|
15
|
+
/** This function makes a fake clock. Its time starts at `start` milliseconds. */
|
|
16
|
+
export declare const fakeClock: (start?: number) => FakeClock;
|
|
17
|
+
/** A fake process receives signals from a test, and keeps each exit code in place of an exit. */
|
|
18
|
+
interface FakeProcess extends ProcessPort {
|
|
19
|
+
/** This method sends a signal to each handler of it. */
|
|
20
|
+
signal(name: ExitSignal): void;
|
|
21
|
+
/** These are the exit codes, in sequence. A correct runtime exits one time. */
|
|
22
|
+
readonly exits: readonly number[];
|
|
23
|
+
/** This count is the number of handlers that are registered. */
|
|
24
|
+
readonly handlers: number;
|
|
25
|
+
}
|
|
26
|
+
/** This function makes a fake process. */
|
|
27
|
+
export declare const fakeProcess: () => FakeProcess;
|
|
28
|
+
/** This function gives a seeded random number generator (mulberry32), with numbers in [0, 1). */
|
|
29
|
+
export declare const seeded: (seed: number) => (() => number);
|
|
30
|
+
//#endregion
|
|
31
|
+
//#region src/contract.d.ts
|
|
32
|
+
/** A contract case is one generated case of the contract test of a declaration. It throws on a failure. */
|
|
33
|
+
interface ContractCase {
|
|
34
|
+
readonly name: string;
|
|
35
|
+
run(): Promise<void>;
|
|
36
|
+
}
|
|
37
|
+
/** These options tune the contract test. */
|
|
38
|
+
interface ContractOptions {
|
|
39
|
+
/** This number is the latency that a degraded dependency adds to each check, in milliseconds. The default of the contract is 100. */
|
|
40
|
+
readonly degradeBy?: number | undefined;
|
|
41
|
+
/** This number is the seed of the jitter. */
|
|
42
|
+
readonly seed?: number | undefined;
|
|
43
|
+
}
|
|
44
|
+
/**
|
|
45
|
+
* This function generates the contract test of a declaration. `declare` gives a new builder with
|
|
46
|
+
* the declaration and the function of each check, each time. The checks must pass at rest. The
|
|
47
|
+
* cases are:
|
|
48
|
+
* - The service at rest: each controller and each dependency is ready, and the service is alive.
|
|
49
|
+
* - Each dependency unavailable: its polls fail at once (the force-fail list). The dependency and
|
|
50
|
+
* each controller that needs it become not ready, and the other controllers stay ready. Liveness
|
|
51
|
+
* stays, unless a liveness check exercises the dependency.
|
|
52
|
+
* - Each dependency degraded: each call of its checks takes the latency longer. Nothing can fail.
|
|
53
|
+
* - Each dependency recovers: after it is unavailable, the service is again at rest.
|
|
54
|
+
* - A negative control: a client that the declaration does not name fails, and nothing changes.
|
|
55
|
+
*
|
|
56
|
+
* A dependency that no check exercises is a finding: the service cannot see it fail.
|
|
57
|
+
*/
|
|
58
|
+
export declare const contractCases: (declare: () => ServiceBuilder, options?: ContractOptions) => ContractCase[];
|
|
59
|
+
//#endregion
|
|
60
|
+
//#region src/testing.d.ts
|
|
61
|
+
/**
|
|
62
|
+
* A behavior is what a scripted check does when the runtime starts it:
|
|
63
|
+
* - `"ok"`, `"error"` and `"indeterminate"` give that outcome at once.
|
|
64
|
+
* - `"throw"` throws at once.
|
|
65
|
+
* - `"timeout"` gives no answer until the runtime aborts the use at the timeout of the poll.
|
|
66
|
+
* - `"hang"` gives no answer, and it ignores the abort, until `heal()`.
|
|
67
|
+
*/
|
|
68
|
+
export type Behavior = Outcome | "throw" | "timeout" | "hang";
|
|
69
|
+
/** A scripted check is a check function whose behavior a test sets. */
|
|
70
|
+
export interface ScriptedCheck {
|
|
71
|
+
readonly run: CheckFn;
|
|
72
|
+
/** This behavior applies to each later call. */
|
|
73
|
+
behavior: Behavior;
|
|
74
|
+
/** This count is the number of calls of the check. */
|
|
75
|
+
readonly calls: number;
|
|
76
|
+
/** This count is the number of calls that are pending. */
|
|
77
|
+
readonly hung: number;
|
|
78
|
+
/** This method ends each hung call with a pass, as when a driver gets its answer late. */
|
|
79
|
+
heal(): void;
|
|
80
|
+
}
|
|
81
|
+
/** This function makes a scripted check with a behavior at the start. */
|
|
82
|
+
export declare const scriptedCheck: (behavior?: Behavior) => ScriptedCheck;
|
|
83
|
+
/** A quiet server is a TCP echo server that can stop its answers and keep its connections open. */
|
|
84
|
+
export interface QuietServer {
|
|
85
|
+
readonly port: number;
|
|
86
|
+
/** This method stops the answers. The connections stay open, and the server reads and discards the data. */
|
|
87
|
+
quiet(): void;
|
|
88
|
+
/** This method starts the answers again. */
|
|
89
|
+
speak(): void;
|
|
90
|
+
/** This count is the number of open connections. */
|
|
91
|
+
readonly connections: number;
|
|
92
|
+
close(): Promise<void>;
|
|
93
|
+
}
|
|
94
|
+
/**
|
|
95
|
+
* This function starts a quiet server on the loopback address. A check against it passes while it
|
|
96
|
+
* answers. When it goes quiet, the check sees a connection that is open but silent. A frozen peer
|
|
97
|
+
* and a black hole that keeps the connection look the same.
|
|
98
|
+
*/
|
|
99
|
+
export declare const quietServer: () => Promise<QuietServer>;
|
|
100
|
+
//#endregion
|
|
101
|
+
export type { ContractCase, ContractOptions, FakeClock, FakeProcess };
|
|
102
|
+
//# sourceMappingURL=testing.d.ts.map
|
package/build/testing.js
ADDED
|
@@ -0,0 +1,349 @@
|
|
|
1
|
+
import { i as recoverSpan, r as failSpan } from "./timing-DxZ4nkQg.js";
|
|
2
|
+
import { valueIn } from "@mark1russell7/quantity";
|
|
3
|
+
import { createServer } from "node:net";
|
|
4
|
+
//#region src/fakes.ts
|
|
5
|
+
/** This function lets the promises that are ready continue, before the clock moves again. */
|
|
6
|
+
const settle = async () => {
|
|
7
|
+
for (let turn = 0; turn < 64; turn += 1) await Promise.resolve();
|
|
8
|
+
};
|
|
9
|
+
/** This function makes a fake clock. Its time starts at `start` milliseconds. */
|
|
10
|
+
const fakeClock = (start = 0) => {
|
|
11
|
+
let now = start;
|
|
12
|
+
let sequence = 0;
|
|
13
|
+
const timers = [];
|
|
14
|
+
const due = (end) => {
|
|
15
|
+
let best;
|
|
16
|
+
for (const timer of timers) if (timer.live && timer.at <= end && (best === void 0 || timer.at < best.at || timer.at === best.at && timer.order < best.order)) best = timer;
|
|
17
|
+
return best;
|
|
18
|
+
};
|
|
19
|
+
return {
|
|
20
|
+
now: () => now,
|
|
21
|
+
after(ms, run) {
|
|
22
|
+
const timer = {
|
|
23
|
+
at: now + Math.max(0, ms),
|
|
24
|
+
order: sequence++,
|
|
25
|
+
run,
|
|
26
|
+
live: true
|
|
27
|
+
};
|
|
28
|
+
timers.push(timer);
|
|
29
|
+
return () => {
|
|
30
|
+
timer.live = false;
|
|
31
|
+
};
|
|
32
|
+
},
|
|
33
|
+
async advance(ms) {
|
|
34
|
+
const end = now + Math.max(0, ms);
|
|
35
|
+
await settle();
|
|
36
|
+
for (;;) {
|
|
37
|
+
const timer = due(end);
|
|
38
|
+
if (timer === void 0) break;
|
|
39
|
+
timer.live = false;
|
|
40
|
+
now = Math.max(now, timer.at);
|
|
41
|
+
timer.run();
|
|
42
|
+
await settle();
|
|
43
|
+
}
|
|
44
|
+
now = Math.max(now, end);
|
|
45
|
+
for (let at = timers.length - 1; at >= 0; at -= 1) if (!timers[at].live) timers.splice(at, 1);
|
|
46
|
+
await settle();
|
|
47
|
+
},
|
|
48
|
+
block(ms) {
|
|
49
|
+
now += Math.max(0, ms);
|
|
50
|
+
},
|
|
51
|
+
get pending() {
|
|
52
|
+
return timers.filter((timer) => timer.live).length;
|
|
53
|
+
}
|
|
54
|
+
};
|
|
55
|
+
};
|
|
56
|
+
/** This function makes a fake process. */
|
|
57
|
+
const fakeProcess = () => {
|
|
58
|
+
const handlers = /* @__PURE__ */ new Map();
|
|
59
|
+
const exits = [];
|
|
60
|
+
return {
|
|
61
|
+
on(signal, handler) {
|
|
62
|
+
const set = handlers.get(signal) ?? /* @__PURE__ */ new Set();
|
|
63
|
+
set.add(handler);
|
|
64
|
+
handlers.set(signal, set);
|
|
65
|
+
return () => {
|
|
66
|
+
set.delete(handler);
|
|
67
|
+
};
|
|
68
|
+
},
|
|
69
|
+
exit(code) {
|
|
70
|
+
exits.push(code);
|
|
71
|
+
},
|
|
72
|
+
signal(name) {
|
|
73
|
+
for (const handler of [...handlers.get(name) ?? []]) handler();
|
|
74
|
+
},
|
|
75
|
+
get exits() {
|
|
76
|
+
return [...exits];
|
|
77
|
+
},
|
|
78
|
+
get handlers() {
|
|
79
|
+
let count = 0;
|
|
80
|
+
for (const set of handlers.values()) count += set.size;
|
|
81
|
+
return count;
|
|
82
|
+
}
|
|
83
|
+
};
|
|
84
|
+
};
|
|
85
|
+
/** This function gives a seeded random number generator (mulberry32), with numbers in [0, 1). */
|
|
86
|
+
const seeded = (seed) => {
|
|
87
|
+
let state = seed >>> 0;
|
|
88
|
+
return () => {
|
|
89
|
+
state = state + 1831565813 >>> 0;
|
|
90
|
+
let t = state;
|
|
91
|
+
t = Math.imul(t ^ t >>> 15, t | 1);
|
|
92
|
+
t ^= t + Math.imul(t ^ t >>> 7, t | 61);
|
|
93
|
+
return ((t ^ t >>> 14) >>> 0) / 4294967296;
|
|
94
|
+
};
|
|
95
|
+
};
|
|
96
|
+
//#endregion
|
|
97
|
+
//#region src/contract.ts
|
|
98
|
+
const ms = (amount) => valueIn(amount, "ms");
|
|
99
|
+
/** The part of a verdict that the contract compares: what a caller and the platform see. */
|
|
100
|
+
const visible = (verdict) => ({
|
|
101
|
+
ready: verdict.ready,
|
|
102
|
+
alive: verdict.alive,
|
|
103
|
+
controllers: verdict.controllers,
|
|
104
|
+
dependencies: verdict.dependencies
|
|
105
|
+
});
|
|
106
|
+
const same = (expected, actual) => JSON.stringify(expected) === JSON.stringify(actual);
|
|
107
|
+
const failure = (what, expected, actual) => /* @__PURE__ */ new Error(`${what}: expected ${JSON.stringify(expected)}, but the verdict was ${JSON.stringify(actual)}`);
|
|
108
|
+
/**
|
|
109
|
+
* This function generates the contract test of a declaration. `declare` gives a new builder with
|
|
110
|
+
* the declaration and the function of each check, each time. The checks must pass at rest. The
|
|
111
|
+
* cases are:
|
|
112
|
+
* - The service at rest: each controller and each dependency is ready, and the service is alive.
|
|
113
|
+
* - Each dependency unavailable: its polls fail at once (the force-fail list). The dependency and
|
|
114
|
+
* each controller that needs it become not ready, and the other controllers stay ready. Liveness
|
|
115
|
+
* stays, unless a liveness check exercises the dependency.
|
|
116
|
+
* - Each dependency degraded: each call of its checks takes the latency longer. Nothing can fail.
|
|
117
|
+
* - Each dependency recovers: after it is unavailable, the service is again at rest.
|
|
118
|
+
* - A negative control: a client that the declaration does not name fails, and nothing changes.
|
|
119
|
+
*
|
|
120
|
+
* A dependency that no check exercises is a finding: the service cannot see it fail.
|
|
121
|
+
*/
|
|
122
|
+
const contractCases = (declare, options = {}) => {
|
|
123
|
+
const degradeBy = options.degradeBy ?? 100;
|
|
124
|
+
const seed = options.seed ?? 1;
|
|
125
|
+
const description = declare().describe();
|
|
126
|
+
const service = description.name;
|
|
127
|
+
const gating = description.checks.filter((check) => check.gates !== null);
|
|
128
|
+
const timingOf = (check) => description.policies[check.gates];
|
|
129
|
+
const failMax = (check) => ms(failSpan(timingOf(check), check.poll, "error").max);
|
|
130
|
+
const recoverMax = (check) => ms(recoverSpan(timingOf(check), check.poll).max);
|
|
131
|
+
const most = (values) => values.reduce((a, b) => Math.max(a, b), 0);
|
|
132
|
+
const settleTime = most(gating.filter((check) => check.gates === "readiness").map(recoverMax)) + 1;
|
|
133
|
+
const quietTime = most(gating.map(failMax)) + 1;
|
|
134
|
+
/** This function starts the declaration on a fake clock, and moves it to rest. */
|
|
135
|
+
const boot = async (extra = () => ({})) => {
|
|
136
|
+
const clock = fakeClock();
|
|
137
|
+
const starting = declare().start({
|
|
138
|
+
clock,
|
|
139
|
+
random: seeded(seed),
|
|
140
|
+
...extra(clock)
|
|
141
|
+
});
|
|
142
|
+
let started = false;
|
|
143
|
+
starting.then(() => started = true, () => started = true);
|
|
144
|
+
for (let step = 0; step < 1e3 && !started; step += 1) await clock.advance(10);
|
|
145
|
+
const health = await starting;
|
|
146
|
+
await clock.advance(settleTime);
|
|
147
|
+
return {
|
|
148
|
+
health,
|
|
149
|
+
clock,
|
|
150
|
+
rest: visible(health.verdict())
|
|
151
|
+
};
|
|
152
|
+
};
|
|
153
|
+
const stop = (health) => health.stop();
|
|
154
|
+
const cases = [{
|
|
155
|
+
name: `${service}: at rest, each controller and each dependency is ready, and the service is alive`,
|
|
156
|
+
async run() {
|
|
157
|
+
const { health, rest } = await boot();
|
|
158
|
+
try {
|
|
159
|
+
const expected = {
|
|
160
|
+
ready: true,
|
|
161
|
+
alive: true,
|
|
162
|
+
controllers: Object.fromEntries(description.controllers.map((each) => [each.name, true])),
|
|
163
|
+
dependencies: Object.fromEntries(description.dependencies.map((each) => [each.name, true]))
|
|
164
|
+
};
|
|
165
|
+
if (!same(expected, rest)) throw failure("at rest", expected, rest);
|
|
166
|
+
} finally {
|
|
167
|
+
stop(health);
|
|
168
|
+
}
|
|
169
|
+
}
|
|
170
|
+
}];
|
|
171
|
+
for (const dependency of description.dependencies) {
|
|
172
|
+
const on = gating.filter((check) => check.dependency === dependency.name);
|
|
173
|
+
const readiness = on.filter((check) => check.gates === "readiness");
|
|
174
|
+
const liveness = on.filter((check) => check.gates === "liveness");
|
|
175
|
+
const unavailable = (rest) => ({
|
|
176
|
+
ready: readiness.length === 0 ? rest.ready : false,
|
|
177
|
+
alive: liveness.length === 0 ? rest.alive : false,
|
|
178
|
+
controllers: Object.fromEntries(description.controllers.map((each) => [each.name, each.requires.includes(dependency.name) && readiness.length > 0 ? false : rest.controllers[each.name]])),
|
|
179
|
+
dependencies: {
|
|
180
|
+
...rest.dependencies,
|
|
181
|
+
[dependency.name]: readiness.length === 0
|
|
182
|
+
}
|
|
183
|
+
});
|
|
184
|
+
cases.push({
|
|
185
|
+
name: `${service}: ${dependency.name} unavailable`,
|
|
186
|
+
async run() {
|
|
187
|
+
if (on.length === 0) throw new Error(`no check that gates a policy exercises ${dependency.name}: the service cannot see it fail`);
|
|
188
|
+
const { health, clock, rest } = await boot();
|
|
189
|
+
try {
|
|
190
|
+
health.forceFail(dependency.name, true);
|
|
191
|
+
await clock.advance(most(on.map(failMax)) + 1);
|
|
192
|
+
const now = visible(health.verdict());
|
|
193
|
+
const expected = unavailable(rest);
|
|
194
|
+
if (!same(expected, now)) throw failure(`${dependency.name} unavailable`, expected, now);
|
|
195
|
+
} finally {
|
|
196
|
+
stop(health);
|
|
197
|
+
}
|
|
198
|
+
}
|
|
199
|
+
});
|
|
200
|
+
cases.push({
|
|
201
|
+
name: `${service}: ${dependency.name} degraded by ${degradeBy} ms, and nothing fails`,
|
|
202
|
+
async run() {
|
|
203
|
+
const tight = on.filter((check) => ms(check.poll.timeout) <= degradeBy);
|
|
204
|
+
if (tight.length > 0) throw new Error(`${tight.map((check) => `${check.name} times out in ${ms(check.poll.timeout)} ms`).join(", ")}: a latency of ${degradeBy} ms fails it`);
|
|
205
|
+
const { health, clock, rest } = await boot((at) => ({ wrapCheck: (check, run) => check.dependency !== dependency.name ? run : (signal) => new Promise((resolve) => at.after(degradeBy, () => resolve(run(signal)))) }));
|
|
206
|
+
try {
|
|
207
|
+
await clock.advance(quietTime + most(on.map((check) => ms(check.poll.interval))));
|
|
208
|
+
const now = visible(health.verdict());
|
|
209
|
+
if (!same(rest, now)) throw failure(`${dependency.name} degraded`, rest, now);
|
|
210
|
+
} finally {
|
|
211
|
+
stop(health);
|
|
212
|
+
}
|
|
213
|
+
}
|
|
214
|
+
});
|
|
215
|
+
cases.push({
|
|
216
|
+
name: `${service}: ${dependency.name} recovers`,
|
|
217
|
+
async run() {
|
|
218
|
+
if (on.length === 0) throw new Error(`no check that gates a policy exercises ${dependency.name}: the service cannot see it recover`);
|
|
219
|
+
const { health, clock, rest } = await boot();
|
|
220
|
+
try {
|
|
221
|
+
health.forceFail(dependency.name, true);
|
|
222
|
+
await clock.advance(most(on.map(failMax)) + 1);
|
|
223
|
+
health.forceFail(dependency.name, false);
|
|
224
|
+
await clock.advance(most(on.map(recoverMax)) + 1);
|
|
225
|
+
const now = visible(health.verdict());
|
|
226
|
+
if (!same(rest, now)) throw failure(`${dependency.name} recovered`, rest, now);
|
|
227
|
+
} finally {
|
|
228
|
+
stop(health);
|
|
229
|
+
}
|
|
230
|
+
}
|
|
231
|
+
});
|
|
232
|
+
}
|
|
233
|
+
const never = `${service}.client-never-registered`;
|
|
234
|
+
cases.push({
|
|
235
|
+
name: `${service}: a client never registered fails, and nothing changes (the negative control)`,
|
|
236
|
+
async run() {
|
|
237
|
+
if (description.dependencies.some((each) => each.name === never)) throw new Error(`the declaration has a dependency named ${never}`);
|
|
238
|
+
const { health, clock, rest } = await boot();
|
|
239
|
+
try {
|
|
240
|
+
health.forceFail(never, true);
|
|
241
|
+
await clock.advance(quietTime);
|
|
242
|
+
const now = visible(health.verdict());
|
|
243
|
+
if (!same(rest, now)) throw failure("the negative control", rest, now);
|
|
244
|
+
} finally {
|
|
245
|
+
stop(health);
|
|
246
|
+
}
|
|
247
|
+
let refused = false;
|
|
248
|
+
try {
|
|
249
|
+
declare().check({
|
|
250
|
+
name: `${never}.check`,
|
|
251
|
+
dependency: never,
|
|
252
|
+
step: "round-trip",
|
|
253
|
+
ownConnection: true,
|
|
254
|
+
gates: null,
|
|
255
|
+
poll: gating[0]?.poll ?? {
|
|
256
|
+
interval: {
|
|
257
|
+
value: 1,
|
|
258
|
+
unit: "s"
|
|
259
|
+
},
|
|
260
|
+
jitter: 0,
|
|
261
|
+
timeout: {
|
|
262
|
+
value: .5,
|
|
263
|
+
unit: "s"
|
|
264
|
+
}
|
|
265
|
+
}
|
|
266
|
+
}).describe();
|
|
267
|
+
} catch {
|
|
268
|
+
refused = true;
|
|
269
|
+
}
|
|
270
|
+
if (!refused) throw new Error(`the declaration accepted a check on ${never}, which is not a dependency`);
|
|
271
|
+
}
|
|
272
|
+
});
|
|
273
|
+
return cases;
|
|
274
|
+
};
|
|
275
|
+
//#endregion
|
|
276
|
+
//#region src/testing.ts
|
|
277
|
+
/** This function makes a scripted check with a behavior at the start. */
|
|
278
|
+
const scriptedCheck = (behavior = "ok") => {
|
|
279
|
+
let calls = 0;
|
|
280
|
+
const hung = /* @__PURE__ */ new Set();
|
|
281
|
+
const script = {
|
|
282
|
+
behavior,
|
|
283
|
+
get calls() {
|
|
284
|
+
return calls;
|
|
285
|
+
},
|
|
286
|
+
get hung() {
|
|
287
|
+
return hung.size;
|
|
288
|
+
},
|
|
289
|
+
heal() {
|
|
290
|
+
for (const end of [...hung]) end();
|
|
291
|
+
},
|
|
292
|
+
run: (signal) => {
|
|
293
|
+
calls += 1;
|
|
294
|
+
const now = script.behavior;
|
|
295
|
+
if (now === "throw") throw new Error("the scripted check threw");
|
|
296
|
+
if (now === "timeout") return new Promise((_, reject) => signal.addEventListener("abort", () => reject(/* @__PURE__ */ new Error("aborted at the timeout"))));
|
|
297
|
+
if (now === "hang") return new Promise((resolve) => {
|
|
298
|
+
const end = () => {
|
|
299
|
+
hung.delete(end);
|
|
300
|
+
resolve("ok");
|
|
301
|
+
};
|
|
302
|
+
hung.add(end);
|
|
303
|
+
});
|
|
304
|
+
return now;
|
|
305
|
+
}
|
|
306
|
+
};
|
|
307
|
+
return script;
|
|
308
|
+
};
|
|
309
|
+
/**
|
|
310
|
+
* This function starts a quiet server on the loopback address. A check against it passes while it
|
|
311
|
+
* answers. When it goes quiet, the check sees a connection that is open but silent. A frozen peer
|
|
312
|
+
* and a black hole that keeps the connection look the same.
|
|
313
|
+
*/
|
|
314
|
+
const quietServer = async () => {
|
|
315
|
+
let silent = false;
|
|
316
|
+
const sockets = /* @__PURE__ */ new Set();
|
|
317
|
+
const server = createServer((socket) => {
|
|
318
|
+
sockets.add(socket);
|
|
319
|
+
socket.on("close", () => sockets.delete(socket));
|
|
320
|
+
socket.on("error", () => sockets.delete(socket));
|
|
321
|
+
socket.on("data", (chunk) => {
|
|
322
|
+
if (!silent) socket.write(chunk);
|
|
323
|
+
});
|
|
324
|
+
});
|
|
325
|
+
await new Promise((resolve, reject) => {
|
|
326
|
+
server.once("error", reject);
|
|
327
|
+
server.listen(0, "127.0.0.1", () => resolve());
|
|
328
|
+
});
|
|
329
|
+
return {
|
|
330
|
+
port: server.address().port,
|
|
331
|
+
quiet: () => {
|
|
332
|
+
silent = true;
|
|
333
|
+
},
|
|
334
|
+
speak: () => {
|
|
335
|
+
silent = false;
|
|
336
|
+
},
|
|
337
|
+
get connections() {
|
|
338
|
+
return sockets.size;
|
|
339
|
+
},
|
|
340
|
+
close: () => new Promise((resolve) => {
|
|
341
|
+
for (const socket of sockets) socket.destroy();
|
|
342
|
+
server.close(() => resolve());
|
|
343
|
+
})
|
|
344
|
+
};
|
|
345
|
+
};
|
|
346
|
+
//#endregion
|
|
347
|
+
export { contractCases, fakeClock, fakeProcess, quietServer, scriptedCheck, seeded };
|
|
348
|
+
|
|
349
|
+
//# sourceMappingURL=testing.js.map
|