@upstash/redis 0.0.0-ci.babdc47f97e036e2863dfed16300cda2b97bf6ad-20240322001434 → 0.0.0-ci.bb93ac2803c2e927dc8bf1a1b3c5aa2a14034eb6-20240930061920

package/pkg/http.ts

@@ -0,0 +1,349 @@
+import { UpstashError, UrlError } from "./error";
+import type { Telemetry } from "./types";
+
+type CacheSetting =
+  | "default"
+  | "force-cache"
+  | "no-cache"
+  | "no-store"
+  | "only-if-cached"
+  | "reload";
+
+export type UpstashRequest = {
+  path?: string[];
+  /**
+   * Request body will be serialized to json
+   */
+  body?: unknown;
+
+  upstashSyncToken?: string;
+};
+export type UpstashResponse<TResult> = { result?: TResult; error?: string };
+
+export interface Requester {
+  /**
+   * When this flag is enabled, any subsequent commands issued by this client are guaranteed to observe the effects of all earlier writes submitted by the same client.
+   */
+  readYourWrites?: boolean;
+
+  /**
+   * This token is used to ensure that the client is in sync with the server. On each request, we send this token in the header, and the server will return a new token.
+   */
+  upstashSyncToken?: string;
+  request: <TResult = unknown>(req: UpstashRequest) => Promise<UpstashResponse<TResult>>;
+}
+
+type ResultError = {
+  result?: string | number | null | (string | number | null)[];
+  error?: string;
+};
+export type RetryConfig =
+  | false
+  | {
+      /**
+       * The number of retries to attempt before giving up.
+       *
+       * @default 5
+       */
+      retries?: number;
+      /**
+       * A backoff function receives the current retry cound and returns a number in milliseconds to wait before retrying.
+       *
+       * @default
+       * ```ts
+       * Math.exp(retryCount) * 50
+       * ```
+       */
+      backoff?: (retryCount: number) => number;
+    };
+
+export type Options = {
+  backend?: string;
+};
+
+export type RequesterConfig = {
+  /**
+   * Configure the retry behaviour in case of network errors
+   */
+  retry?: RetryConfig;
+
+  /**
+   * Due to the nature of dynamic and custom data, it is possible to write data to redis that is not
+   * valid json and will therefore cause errors when deserializing. This used to happen very
+   * frequently with non-utf8 data, such as emojis.
+   *
+   * By default we will therefore encode the data as base64 on the server, before sending it to the
+   * client. The client will then decode the base64 data and parse it as utf8.
+   *
+   * For very large entries, this can add a few milliseconds, so if you are sure that your data is
+   * valid utf8, you can disable this behaviour by setting this option to false.
+   *
+   * Here's what the response body looks like:
+   *
+   * ```json
+   * {
+   *  result?: "base64-encoded",
+   *  error?: string
+   * }
+   * ```
+   *
+   * @default "base64"
+   */
+  responseEncoding?: false | "base64";
+
+  /**
+   * Configure the cache behaviour
+   * @default "no-store"
+   */
+  cache?: CacheSetting;
+};
+
+export type HttpClientConfig = {
+  headers?: Record<string, string>;
+  baseUrl: string;
+  options?: Options;
+  retry?: RetryConfig;
+  agent?: any;
+  signal?: AbortSignal;
+  keepAlive?: boolean;
+
+  /**
+   * When this flag is enabled, any subsequent commands issued by this client are guaranteed to observe the effects of all earlier writes submitted by the same client.
+   */
+  readYourWrites?: boolean;
+} & RequesterConfig;
+
+export class HttpClient implements Requester {
+  public baseUrl: string;
+  public headers: Record<string, string>;
+
+  public readonly options: {
+    backend?: string;
+    agent: any;
+    signal?: AbortSignal;
+    responseEncoding?: false | "base64";
+    cache?: CacheSetting;
+    keepAlive: boolean;
+  };
+  public readYourWrites: boolean;
+  public upstashSyncToken = "";
+
+  public readonly retry: {
+    attempts: number;
+    backoff: (retryCount: number) => number;
+  };
+
+  public constructor(config: HttpClientConfig) {
+    this.options = {
+      backend: config.options?.backend,
+      agent: config.agent,
+      responseEncoding: config.responseEncoding ?? "base64", // default to base64
+      cache: config.cache,
+      signal: config.signal,
+      keepAlive: config.keepAlive ?? true,
+    };
+    this.upstashSyncToken = "";
+    this.readYourWrites = config.readYourWrites ?? true;
+
+    this.baseUrl = config.baseUrl.replace(/\/$/, "");
+
+    /**
+     * regex to check if the baseUrl starts with http:// or https://
+     * - `^` asserts the position at the start of the string.
+     * - `[^\s/$.?#]` makes sure that the domain starts correctly;
+     *   without white space, '/', '$', '.', '?' or '#'
+     * - `.` matches any character except new line
+     * - `[^\s]*` matches anything except white space
+     * - `$` asserts the position at the end of the string.
+     */
+    const urlRegex = /^https?:\/\/[^\s#$./?].\S*$/;
+    if (!urlRegex.test(this.baseUrl)) {
+      throw new UrlError(this.baseUrl);
+    }
+
+    this.headers = {
+      "Content-Type": "application/json",
+
+      ...config.headers,
+    };
+
+    if (this.options.responseEncoding === "base64") {
+      this.headers["Upstash-Encoding"] = "base64";
+    }
+
+    this.retry =
+      typeof config.retry === "boolean" && !config.retry
+        ? {
+            attempts: 1,
+            backoff: () => 0,
+          }
+        : {
+            attempts: config.retry?.retries ?? 5,
+            backoff: config.retry?.backoff ?? ((retryCount) => Math.exp(retryCount) * 50),
+          };
+  }
+
+  public mergeTelemetry(telemetry: Telemetry): void {
+    this.headers = merge(this.headers, "Upstash-Telemetry-Runtime", telemetry.runtime);
+    this.headers = merge(this.headers, "Upstash-Telemetry-Platform", telemetry.platform);
+    this.headers = merge(this.headers, "Upstash-Telemetry-Sdk", telemetry.sdk);
+  }
+
+  public async request<TResult>(req: UpstashRequest): Promise<UpstashResponse<TResult>> {
+    const requestOptions: RequestInit & { backend?: string; agent?: any } = {
+      //@ts-expect-error this should throw due to bun regression
+      cache: this.options.cache,
+      method: "POST",
+      headers: this.headers,
+      body: JSON.stringify(req.body),
+      keepalive: this.options.keepAlive,
+      agent: this.options.agent,
+      signal: this.options.signal,
+
+      /**
+       * Fastly specific
+       */
+      backend: this.options.backend,
+    };
+
+    /**
+     * We've recieved a new `upstash-sync-token` in the previous response. We use it in the next request to observe the effects of previous requests.
+     */
+    if (this.readYourWrites) {
+      const newHeader = this.upstashSyncToken;
+      this.headers["upstash-sync-token"] = newHeader;
+    }
+
+    let res: Response | null = null;
+    let error: Error | null = null;
+    for (let i = 0; i <= this.retry.attempts; i++) {
+      try {
+        res = await fetch([this.baseUrl, ...(req.path ?? [])].join("/"), requestOptions);
+        break;
+      } catch (error_) {
+        if (this.options.signal?.aborted) {
+          const myBlob = new Blob([
+            JSON.stringify({ result: this.options.signal.reason ?? "Aborted" }),
+          ]);
+          const myOptions = {
+            status: 200,
+            statusText: this.options.signal.reason ?? "Aborted",
+          };
+          res = new Response(myBlob, myOptions);
+          break;
+        }
+        error = error_ as Error;
+        await new Promise((r) => setTimeout(r, this.retry.backoff(i)));
+      }
+    }
+    if (!res) {
+      throw error ?? new Error("Exhausted all retries");
+    }
+
+    const body = (await res.json()) as UpstashResponse<string>;
+    if (!res.ok) {
+      throw new UpstashError(`${body.error}, command was: ${JSON.stringify(req.body)}`);
+    }
+
+    if (this.readYourWrites) {
+      const headers = res.headers;
+      this.upstashSyncToken = headers.get("upstash-sync-token") ?? "";
+    }
+
+    /**
+     * We save the new `upstash-sync-token` in the response header to use it in the next request.
+     */
+    if (this.readYourWrites) {
+      const headers = res.headers;
+      this.upstashSyncToken = headers.get("upstash-sync-token") ?? "";
+    }
+
+    if (this.options.responseEncoding === "base64") {
+      if (Array.isArray(body)) {
+        return body.map(({ result, error }) => ({
+          result: decode(result),
+          error,
+        })) as UpstashResponse<TResult>;
+      }
+      const result = decode(body.result) as any;
+      return { result, error: body.error };
+    }
+
+    return body as UpstashResponse<TResult>;
+  }
+}
+
+function base64decode(b64: string): string {
+  let dec = "";
+  try {
+    /**
+     * Using only atob() is not enough because it doesn't work with unicode characters
+     */
+    const binString = atob(b64);
+    const size = binString.length;
+    const bytes = new Uint8Array(size);
+    for (let i = 0; i < size; i++) {
+      // eslint-disable-next-line unicorn/prefer-code-point
+      bytes[i] = binString.charCodeAt(i);
+    }
+    dec = new TextDecoder().decode(bytes);
+  } catch {
+    dec = b64;
+  }
+  return dec;
+  // try {
+  //   return decodeURIComponent(dec);
+  // } catch {
+  //   return dec;
+  // }
+}
+
+function decode(raw: ResultError["result"]): ResultError["result"] {
+  let result: any = undefined;
+  switch (typeof raw) {
+    case "undefined": {
+      return raw;
+    }
+
+    case "number": {
+      result = raw;
+      break;
+    }
+    case "object": {
+      // eslint-disable-next-line unicorn/prefer-ternary
+      if (Array.isArray(raw)) {
+        result = raw.map((v) =>
+          typeof v === "string"
+            ? base64decode(v)
+            : Array.isArray(v)
+              ? v.map((element) => decode(element))
+              : v
+        );
+      } else {
+        // If it's not an array it must be null
+        // Apparently null is an object in javascript
+        result = null;
+      }
+      break;
+    }
+
+    case "string": {
+      result = raw === "OK" ? "OK" : base64decode(raw);
+      break;
+    }
+
+    default: {
+      break;
+    }
+  }
+
+  return result;
+}
+
+function merge(obj: Record<string, string>, key: string, value?: string): Record<string, string> {
+  if (!value) {
+    return obj;
+  }
+  obj[key] = obj[key] ? [obj[key], value].join(",") : value;
+  return obj;
+}

package/pkg/index.ts

@@ -0,0 +1 @@
+export * from "./error";

package/pkg/pipeline.test.ts

@@ -0,0 +1,254 @@
+import { Pipeline } from "./pipeline";
+import { Redis } from "./redis";
+import { keygen, newHttpClient, randomID } from "./test-utils";
+
+import { afterEach, describe, expect, test } from "bun:test";
+import { ScriptLoadCommand } from "./commands/script_load";
+
+const client = newHttpClient();
+
+const { newKey, cleanup } = keygen();
+afterEach(cleanup);
+
+describe("with destructuring", () => {
+  test("correctly binds this", async () => {
+    const { pipeline } = new Redis(client);
+    const p = pipeline();
+
+    const { echo, exec } = p;
+    echo("Hello");
+
+    const res = await exec();
+    expect(res).toEqual(["Hello"]);
+  });
+});
+
+describe("with single command", () => {
+  test("works with multiple commands", async () => {
+    const p = new Pipeline({ client });
+    p.set(newKey(), randomID());
+    const res = await p.exec();
+    expect(res.length).toEqual(1);
+    expect(res[0]).toEqual("OK");
+  });
+});
+
+describe("when chaining in a for loop", () => {
+  test("works", async () => {
+    const key = newKey();
+    const res = await new Pipeline({ client }).set(key, randomID()).get(key).exec();
+
+    expect(res.length).toEqual(2);
+  });
+});
+
+describe("when chaining inline", () => {
+  test("works", async () => {
+    const key = newKey();
+    const p = new Pipeline({ client });
+    for (let i = 0; i < 10; i++) {
+      p.set(key, randomID());
+    }
+
+    const res = await p.exec();
+    expect(res.length).toEqual(10);
+  });
+});
+
+describe("when no commands were added", () => {
+  test("throws", async () => {
+    let hasThrown = false;
+    await new Pipeline({ client }).exec().catch(() => {
+      hasThrown = true;
+    });
+    expect(hasThrown).toBeTrue();
+  });
+});
+
+describe("when length called", () => {
+  test("before exec()", () => {
+    const key = newKey();
+    const p = new Pipeline({ client });
+    for (let i = 0; i < 10; i++) {
+      p.set(key, randomID());
+    }
+    expect(p.length()).toEqual(10);
+  });
+
+  test("after exec()", async () => {
+    const key = newKey();
+    const p = new Pipeline({ client });
+    for (let i = 0; i < 10; i++) {
+      p.set(key, randomID());
+    }
+    await p.exec();
+    expect(p.length()).toEqual(10);
+  });
+});
+
+describe("when one command throws an error", () => {
+  test("throws", async () => {
+    const p = new Pipeline({ client }).set("key", "value").hget("key", "field");
+    let hasThrown = false;
+    await p.exec().catch(() => {
+      hasThrown = true;
+    });
+    expect(hasThrown).toBeTrue();
+  });
+});
+
+describe("transaction", () => {
+  test("works", async () => {
+    const key = newKey();
+    const value = randomID();
+    const tx = new Pipeline({ client, multiExec: true });
+    tx.set(key, value);
+    tx.get(key);
+    tx.del(key);
+
+    const [ok, storedvalue, deleted] = await tx.exec<["OK", string, number]>();
+    expect(ok).toEqual("OK");
+    expect(storedvalue).toEqual(value);
+    expect(deleted).toEqual(1);
+  });
+});
+describe("use all the things", () => {
+  test("works", async () => {
+    const p = new Pipeline({ client });
+
+    const persistentKey = newKey();
+    const persistentKey2 = newKey();
+
+    const scriptHash = await new ScriptLoadCommand(["return 1"]).exec(client);
+
+    p.append(newKey(), "hello")
+      .bitcount(newKey(), 0, 1)
+      .bitfield(newKey())
+      .set("u4", "#0", 15)
+      .get("u4", "#0")
+      .overflow("WRAP")
+      .incrby("u4", "#0", 10)
+      .exec()
+      .bitop("and", newKey(), newKey())
+      .bitpos(newKey(), 1, 0)
+      .dbsize()
+      .decr(newKey())
+      .decrby(newKey(), 1)
+      .del(newKey())
+      .echo("hello")
+      .eval("return ARGV[1]", [], ["Hello"])
+      .evalsha(scriptHash, [], ["Hello"])
+      .exists(newKey())
+      .expire(newKey(), 5)
+      .expireat(newKey(), Math.floor(Date.now() / 1000) + 60)
+      .flushall()
+      .flushdb()
+      .get(newKey())
+      .getbit(newKey(), 0)
+      .getdel(newKey())
+      .getset(newKey(), "hello")
+      .hdel(newKey(), "field")
+      .hexists(newKey(), "field")
+      .hget(newKey(), "field")
+      .hgetall(newKey())
+      .hincrby(newKey(), "field", 1)
+      .hincrbyfloat(newKey(), "field", 1.5)
+      .hkeys(newKey())
+      .hlen(newKey())
+      .hmget(newKey(), newKey())
+      .hmset(newKey(), { field: "field", value: "value" })
+      .hscan(newKey(), 0)
+      .hset(newKey(), { field: "value" })
+      .hsetnx(newKey(), "field", "value")
+      .hstrlen(newKey(), "field")
+      .hvals(newKey())
+      .incr(newKey())
+      .incrby(newKey(), 1)
+      .incrbyfloat(newKey(), 1.5)
+      .keys("*")
+      .lindex(newKey(), 0)
+      .linsert(newKey(), "before", "pivot", "value")
+      .llen(newKey())
+      .lmove(newKey(), newKey(), "left", "right")
+      .lpop(newKey())
+      .lpos(newKey(), "value")
+      .lpush(persistentKey, "element")
+      .lpushx(newKey(), "element1", "element2")
+      .lrange(newKey(), 0, 1)
+      .lrem(newKey(), 1, "value")
+      .lset(persistentKey, 0, "value")
+      .ltrim(newKey(), 0, 1)
+      .hrandfield(newKey())
+      .hrandfield(newKey(), 2)
+      .hrandfield(newKey(), 3, true)
+      .mget<[string, string]>(newKey(), newKey())
+      .mset({ key1: "value", key2: "value" })
+      .msetnx({ key3: "value", key4: "value" })
+      .persist(newKey())
+      .pexpire(newKey(), 1000)
+      .pexpireat(newKey(), Date.now() + 1000)
+      .ping()
+      .psetex(newKey(), 1, "value")
+      .pttl(newKey())
+      .publish("test", "hello")
+      .randomkey()
+      .rename(persistentKey, persistentKey2)
+      .renamenx(persistentKey2, newKey())
+      .rpop(newKey())
+      .rpush(newKey(), "element1", "element2")
+      .rpushx(newKey(), "element1", "element2")
+      .sadd(newKey(), "memeber1", "member2")
+      .scan(0)
+      .scard(newKey())
+      .sdiff(newKey())
+      .sdiffstore(newKey(), newKey())
+      .set(newKey(), "value")
+      .setbit(newKey(), 1, 1)
+      .setex(newKey(), 1, "value")
+      .setnx(newKey(), "value")
+      .setrange(newKey(), 1, "value")
+      .sinter(newKey(), newKey())
+      .sinterstore(newKey(), newKey())
+      .sismember(newKey(), "member")
+      .smembers(newKey())
+      .smove(newKey(), newKey(), "member")
+      .spop(newKey())
+      .srandmember(newKey())
+      .srem(newKey(), "member")
+      .sscan(newKey(), 0)
+      .strlen(newKey())
+      .sunion(newKey())
+      .sunionstore(newKey(), newKey())
+      .time()
+      .touch(newKey())
+      .ttl(newKey())
+      .type(newKey())
+      .unlink(newKey())
+      .zadd(newKey(), { score: 0, member: "member" })
+      .zcard(newKey())
+      .scriptExists(scriptHash)
+      .scriptFlush({ async: true })
+      .scriptLoad("return 1")
+      .zcount(newKey(), 0, 1)
+      .zincrby(newKey(), 1, "member")
+      .zinterstore(newKey(), 1, [newKey()])
+      .zlexcount(newKey(), "-", "+")
+      .zpopmax(newKey())
+      .zpopmin(newKey())
+      .zrange(newKey(), 0, 1)
+      .zrank(newKey(), "member")
+      .zrem(newKey(), "member")
+      .zremrangebylex(newKey(), "-", "+")
+      .zremrangebyrank(newKey(), 0, 1)
+      .zremrangebyscore(newKey(), 0, 1)
+      .zrevrank(newKey(), "member")
+      .zscan(newKey(), 0)
+      .zscore(newKey(), "member")
+      .zunionstore(newKey(), 1, [newKey()])
+      .zunion(1, [newKey()])
+      .json.set(newKey(), "$", { hello: "world" });
+
+    const res = await p.exec();
+    expect(res.length).toEqual(121);
+  });
+});