@instantdb/resumable-stream 0.0.0

package/.tshy/build.json

@@ -0,0 +1,8 @@
+{
+  "extends": "../tsconfig.json",
+  "compilerOptions": {
+    "rootDir": "../src",
+    "module": "nodenext",
+    "moduleResolution": "nodenext"
+  }
+}

package/.tshy/commonjs.json

@@ -0,0 +1,16 @@
+{
+  "extends": "./build.json",
+  "include": [
+    "../src/**/*.ts",
+    "../src/**/*.cts",
+    "../src/**/*.tsx",
+    "../src/**/*.json"
+  ],
+  "exclude": [
+    "../src/**/*.mts",
+    "../src/package.json"
+  ],
+  "compilerOptions": {
+    "outDir": "../.tshy-build/commonjs"
+  }
+}

package/.tshy/esm.json

@@ -0,0 +1,15 @@
+{
+  "extends": "./build.json",
+  "include": [
+    "../src/**/*.ts",
+    "../src/**/*.mts",
+    "../src/**/*.tsx",
+    "../src/**/*.json"
+  ],
+  "exclude": [
+    "../src/package.json"
+  ],
+  "compilerOptions": {
+    "outDir": "../.tshy-build/esm"
+  }
+}

package/.turbo/turbo-build.log

@@ -0,0 +1,46 @@
+
+
+> @instantdb/resumable-stream@0.0.0 build /Users/daniel/projects/instant-private/client/packages/resumable-stream
+> rm -rf dist; npm run build:tshy && npm run build:standalone && npm run check-exports
+
+
+> @instantdb/resumable-stream@0.0.0 build:tshy
+> tshy
+
+⠙
+> @instantdb/resumable-stream@0.0.0 build:standalone
+> vite build
+
+vite v5.4.14 building for production...
+transforming (1) src/index.tstransforming (51) ../../node_modules/.pnpm/uuid@11.1.0/node_modules/uuid/dist/esm-browser/md5.js✓ 73 modules transformed.
+rendering chunks (1)...computing gzip size (0)...computing gzip size (1)...dist/standalone/index.umd.cjs  66.65 kB │ gzip: 20.91 kB
+rendering chunks (1)...computing gzip size (1)...dist/standalone/index.js  105.81 kB │ gzip: 27.65 kB
+✓ built in 390ms
+⠙
+> @instantdb/resumable-stream@0.0.0 check-exports
+> attw --pack .
+
+
+@instantdb/resumable-stream v0.0.0
+
+Build tools:
+- @arethetypeswrong/cli@^0.17.4
+- typescript@^5.9.3
+- vite@^5.2.0
+- tshy@^3.0.2
+
+ No problems found 🌟
+
+
+┌───────────────────┬────────────────────────────────────────────┬───────────────────────────────┐
+│                   │ "@instantdb/resumable-stream/package.json" │ "@instantdb/resumable-stream" │
+├───────────────────┼────────────────────────────────────────────┼───────────────────────────────┤
+│ node10            │ 🟢 (JSON)                                  │ 🟢                            │
+├───────────────────┼────────────────────────────────────────────┼───────────────────────────────┤
+│ node16 (from CJS) │ 🟢 (JSON)                                  │ 🟢 (CJS)                      │
+├───────────────────┼────────────────────────────────────────────┼───────────────────────────────┤
+│ node16 (from ESM) │ 🟢 (JSON)                                  │ 🟢 (ESM)                      │
+├───────────────────┼────────────────────────────────────────────┼───────────────────────────────┤
+│ bundler           │ 🟢 (JSON)                                  │ 🟢                            │
+└───────────────────┴────────────────────────────────────────────┴───────────────────────────────┘
+⠙

package/.turbo/turbo-test$colon$ci.log

@@ -0,0 +1,14 @@
+
+
+> @instantdb/resumable-stream@0.0.0 test:ci /Users/daniel/projects/instant-private/client/packages/resumable-stream
+> vitest run
+
+
+ RUN  v1.6.1 /Users/daniel/projects/instant-private/client/packages/resumable-stream
+
+include: **/*.{test,spec}.?(c|m)[jt]s?(x)
+exclude:  **/node_modules/**, **/dist/**, **/cypress/**, **/.{idea,git,cache,output,temp}/**, **/{karma,rollup,webpack,vite,vitest,jest,ava,babel,nyc,cypress,tsup,build,eslint,prettier}.config.*
+watch exclude:  **/node_modules/**, **/dist/**
+
+No test files found, exiting with code 1
+ ELIFECYCLE  Command failed with exit code 1.

package/README.md

@@ -0,0 +1,112 @@
+<p align="center">
+  <a href="https://instantdb.com">
+    <img alt="Shows the Instant logo" src="https://instantdb.com/img/icon/android-chrome-512x512.png" width="10%">
+  </a>
+  <h1 align="center">@instantdb/resumable-stream</h1>
+</p>
+
+<p align="center">
+  <a
+    href="https://discord.com/invite/VU53p7uQcE" >
+    <img height=20 src="https://img.shields.io/discord/1031957483243188235" />
+  </a>
+  <img src="https://img.shields.io/github/stars/instantdb/instant" alt="stars">
+</p>
+
+<p align="center">
+   <a href="https://www.instantdb.com/docs/start-vanilla">Get Started</a> ·
+   <a href="https://instantdb.com/examples">Examples</a> ·
+   <a href="https://www.instantdb.com/docs/start-vanilla">Docs</a> ·
+   <a href="https://discord.com/invite/VU53p7uQcE">Discord</a>
+<p>
+
+Welcome to [Instant's](http://instantdb.com) resumable-stream library.
+
+This is a drop-in replacement for Vercel's resumable-stream library using InstantDB streams.
+
+Instant's streams have no dependency on Redis and they never expire.
+
+## Usage
+
+You can provide your appId and adminToken as arguments to `createResumableStreamContext` or export `INSTANT_APP_ID` and `INSTANT_APP_ADMIN_TOKEN`.
+
+### Idempotent API
+
+```typescript
+import { createResumableStreamContext } from '@instantdb/resumable-stream';
+import { after } from 'next/server';
+
+const streamContext = createResumableStreamContext({
+  waitUntil: after,
+  appId: YOUR_INSTANT_APP_ID, // or export INSTANT_APP_ID
+  adminToken: YOUR_INSTANT_APP_ADMIN_TOKEN, // or export INSTANT_APP_ADMIN_TOKEN
+});
+
+export async function GET(
+  req: NextRequest,
+  { params }: { params: Promise<{ streamId: string }> },
+) {
+  const { streamId } = await params;
+  const resumeAt = req.nextUrl.searchParams.get('resumeAt');
+  const stream = await streamContext.resumableStream(
+    streamId,
+    makeTestStream,
+    resumeAt ? parseInt(resumeAt) : undefined,
+  );
+  return new Response(stream, {
+    headers: {
+      'Content-Type': 'text/event-stream',
+    },
+  });
+}
+```
+
+### Usage with explicit resumption
+
+```typescript
+import { createResumableStreamContext } from 'resumable-stream';
+import { after } from 'next/server';
+
+const streamContext = createResumableStreamContext({
+  waitUntil: after,
+  appId: YOUR_INSTANT_APP_ID, // or export INSTANT_APP_ID
+  adminToken: YOUR_INSTANT_APP_ADMIN_TOKEN, // or export INSTANT_APP_ADMIN_TOKEN
+});
+
+export async function POST(
+  req: NextRequest,
+  { params }: { params: Promise<{ streamId: string }> },
+) {
+  const { streamId } = await params;
+  const stream = await streamContext.createNewResumableStream(
+    streamId,
+    makeTestStream,
+  );
+  return new Response(stream, {
+    headers: {
+      'Content-Type': 'text/event-stream',
+    },
+  });
+}
+
+export async function GET(
+  req: NextRequest,
+  { params }: { params: Promise<{ streamId: string }> },
+) {
+  const { streamId } = await params;
+  const resumeAt = req.nextUrl.searchParams.get('resumeAt');
+  const stream = await streamContext.resumeExistingStream(
+    streamId,
+    resumeAt ? parseInt(resumeAt) : undefined,
+  );
+  return new Response(stream, {
+    headers: {
+      'Content-Type': 'text/event-stream',
+    },
+  });
+}
+```
+
+# Questions?
+
+If you have any questions, feel free to drop us a line on our [Discord](https://discord.com/invite/VU53p7uQcE)

package/__tests__/src/resumable-stream.test.ts

@@ -0,0 +1,341 @@
+import { beforeAll, beforeEach, describe, expect, it } from 'vitest';
+import {
+  createResumableStreamContext,
+  type ResumableStreamContext,
+} from '../../src/index.js';
+import { createTestingStream, streamToBuffer } from './testing-stream.js';
+
+type EphemeralApp = {
+  appId: string;
+  adminToken: string;
+  apiURI: string;
+};
+
+async function createEphemeralAppForTests(): Promise<EphemeralApp> {
+  const apiURI = process.env.INSTANT_API_URI || 'https://api.instantdb.com';
+  const response = await fetch(`${apiURI}/dash/apps/ephemeral`, {
+    method: 'POST',
+    headers: {
+      'Content-Type': 'application/json',
+    },
+    body: JSON.stringify({
+      title: `resumable-stream-tests-${Date.now()}`,
+    }),
+  });
+
+  if (!response.ok) {
+    const body = await response.text();
+    throw new Error(
+      `Failed to create ephemeral app (${response.status}): ${body}`,
+    );
+  }
+
+  const json = (await response.json()) as any;
+  const appId = json?.app?.id ?? json?.appId;
+  const adminToken =
+    json?.app?.['admin-token'] ??
+    json?.app?.adminToken ??
+    json?.app?.admin_token ??
+    json?.adminToken;
+
+  if (!appId || !adminToken) {
+    throw new Error(
+      `Unexpected ephemeral app response: ${JSON.stringify(json)}`,
+    );
+  }
+
+  return {
+    appId,
+    adminToken,
+    apiURI,
+  };
+}
+
+function withScopedStreamIds(
+  context: ResumableStreamContext,
+  scope: string,
+): ResumableStreamContext {
+  const scopedId = (streamId: string) => `${scope}:${streamId}`;
+  return {
+    resumableStream: (streamId, makeStream, skipCharacters) =>
+      context.resumableStream(scopedId(streamId), makeStream, skipCharacters),
+    resumeExistingStream: (streamId, skipCharacters) =>
+      context.resumeExistingStream(scopedId(streamId), skipCharacters),
+    createNewResumableStream: (streamId, makeStream, skipCharacters) =>
+      context.createNewResumableStream(
+        scopedId(streamId),
+        makeStream,
+        skipCharacters,
+      ),
+    hasExistingStream: (streamId) =>
+      context.hasExistingStream(scopedId(streamId)),
+  };
+}
+
+describe('resumable stream', () => {
+  let app: EphemeralApp;
+  let resume: ResumableStreamContext;
+  let waitUntilPromises: Promise<any>[] = [];
+
+  beforeAll(async () => {
+    app = await createEphemeralAppForTests();
+  }, 60000);
+
+  beforeEach(() => {
+    const baseContext = createResumableStreamContext({
+      waitUntil: (p) => waitUntilPromises.push(p),
+      appId: app.appId,
+      adminToken: app.adminToken,
+      apiURI: app.apiURI,
+    });
+    resume = withScopedStreamIds(baseContext, crypto.randomUUID());
+  });
+
+  it('should act like a normal stream', async () => {
+    const { readable, writer } = createTestingStream();
+    const stream = await resume.resumableStream('test', () => readable);
+    writer.write('1\n');
+    writer.write('2\n');
+    writer.write('3\n');
+    writer.close();
+    const result2 = await streamToBuffer(stream);
+    expect(result2).toEqual('1\n2\n3\n');
+  });
+
+  it('should resume a done stream', async () => {
+    const { readable, writer } = createTestingStream();
+    const stream = await resume.resumableStream('test', () => readable);
+    const stream2 = await resume.resumableStream('test', () => readable);
+    writer.write('1\n');
+    writer.write('2\n');
+    writer.close();
+    const result = await streamToBuffer(stream);
+    const result2 = await streamToBuffer(stream2);
+    expect(result).toEqual('1\n2\n');
+    expect(result2).toEqual('1\n2\n');
+  });
+
+  it('hasExistingStream', async () => {
+    const { readable, writer } = createTestingStream();
+    expect(await resume.hasExistingStream('test')).toBe(null);
+    const stream = await resume.resumableStream('test', () => readable);
+    expect(await resume.hasExistingStream('test')).toBe(true);
+    expect(await resume.hasExistingStream('test2')).toBe(null);
+    const stream2 = await resume.resumableStream('test', () => readable);
+    expect(await resume.hasExistingStream('test')).toBe(true);
+    writer.write('1\n');
+    writer.write('2\n');
+    writer.close();
+
+    const result = await streamToBuffer(stream);
+    const result2 = await streamToBuffer(stream2);
+    expect(result).toEqual('1\n2\n');
+    expect(result2).toEqual('1\n2\n');
+    await Promise.all(waitUntilPromises);
+    expect(await resume.hasExistingStream('test')).toBe('DONE');
+  });
+
+  it('should resume a done stream reverse read', async () => {
+    const { readable, writer } = createTestingStream();
+    const stream = await resume.resumableStream('test', () => readable);
+    const stream2 = await resume.resumableStream('test', () => readable);
+    writer.write('1\n');
+    writer.write('2\n');
+    writer.close();
+    const result2 = await streamToBuffer(stream2);
+    const result = await streamToBuffer(stream);
+
+    expect(result).toEqual('1\n2\n');
+    expect(result2).toEqual('1\n2\n');
+  });
+
+  it('should resume an in-progress stream', async () => {
+    const { readable, writer } = createTestingStream();
+    const stream = await resume.resumableStream('test', () => readable);
+    writer.write('1\n');
+    const stream2 = await resume.resumableStream('test', () => readable);
+    writer.write('2\n');
+    writer.close();
+    const result = await streamToBuffer(stream);
+    const result2 = await streamToBuffer(stream2);
+    expect(result).toEqual('1\n2\n');
+    expect(result2).toEqual('1\n2\n');
+  });
+
+  it('should actually stream', async () => {
+    const { readable, writer } = createTestingStream();
+    const stream = await resume.resumableStream('test', () => readable);
+    writer.write('1\n');
+    const stream2 = await resume.resumableStream('test', () => readable);
+    const result = await streamToBuffer(stream, 1);
+    const result2 = await streamToBuffer(stream2, 1);
+    expect(result).toEqual('1\n');
+    expect(result2).toEqual('1\n');
+    writer.write('2\n');
+    writer.close();
+    const step1 = await streamToBuffer(stream);
+    const step2 = await streamToBuffer(stream2);
+    expect(step1).toEqual('2\n');
+    expect(step2).toEqual('2\n');
+  });
+
+  it('should actually stream producer first', async () => {
+    const { readable, writer } = createTestingStream();
+    const stream = await resume.resumableStream('test', () => readable);
+    writer.write('1\n');
+    const stream2 = await resume.resumableStream('test', () => readable);
+    const result = await streamToBuffer(stream, 1);
+    expect(result).toEqual('1\n');
+    writer.write('2\n');
+    writer.close();
+    const step1 = await streamToBuffer(stream);
+    const step2 = await streamToBuffer(stream2);
+    expect(step1).toEqual('2\n');
+    expect(step2).toEqual('1\n2\n');
+  });
+
+  it('should actually stream consumer first', async () => {
+    const { readable, writer } = createTestingStream();
+    const stream = await resume.resumableStream('test', () => readable);
+    writer.write('1\n');
+    const stream2 = await resume.resumableStream('test', () => readable);
+    const result2 = await streamToBuffer(stream2, 1);
+    expect(result2).toEqual('1\n');
+    writer.write('2\n');
+    writer.close();
+    const step1 = await streamToBuffer(stream);
+    const step2 = await streamToBuffer(stream2);
+    expect(step1).toEqual('1\n2\n');
+    expect(step2).toEqual('2\n');
+  });
+
+  it('should resume multiple streams', async () => {
+    const { readable, writer } = createTestingStream();
+    const stream = await resume.resumableStream('test', () => readable);
+    writer.write('1\n');
+    const stream2 = await resume.resumableStream('test', () => readable);
+    writer.write('2\n');
+    const stream3 = await resume.resumableStream('test', () => readable);
+    writer.close();
+    const result = await streamToBuffer(stream);
+    const result2 = await streamToBuffer(stream2);
+    const result3 = await streamToBuffer(stream3);
+    expect(result).toEqual('1\n2\n');
+    expect(result2).toEqual('1\n2\n');
+    expect(result3).toEqual('1\n2\n');
+  });
+
+  it('should differentiate between streams', async () => {
+    const { readable, writer } = createTestingStream();
+    const { readable: readable2, writer: writer2 } = createTestingStream();
+    const stream1 = await resume.resumableStream('test', () => readable);
+    const stream2 = await resume.resumableStream('test2', () => readable2);
+    const stream12 = await resume.resumableStream('test', () => readable);
+    const stream22 = await resume.resumableStream('test2', () => readable2);
+    writer.write('1\n');
+    writer.write('2\n');
+    writer.close();
+    writer2.write('writer2\n');
+    writer2.close();
+    const result1 = await streamToBuffer(stream1);
+    const result2 = await streamToBuffer(stream2);
+    const result12 = await streamToBuffer(stream12);
+    const result22 = await streamToBuffer(stream22);
+    expect(result1).toEqual('1\n2\n');
+    expect(result2).toEqual('writer2\n');
+    expect(result12).toEqual('1\n2\n');
+    expect(result22).toEqual('writer2\n');
+  });
+
+  it('should respect skipCharacters', async () => {
+    const { readable, writer } = createTestingStream();
+    const stream = await resume.resumableStream('test', () => readable);
+    writer.write('1\n');
+    writer.write('2\n');
+    const stream2 = await resume.resumableStream('test', () => readable, 2);
+    writer.close();
+    const result = await streamToBuffer(stream);
+    const result2 = await streamToBuffer(stream2);
+    expect(result).toEqual('1\n2\n');
+    expect(result2).toEqual('2\n');
+  });
+
+  it('should respect skipCharacters 2', async () => {
+    const { readable, writer } = createTestingStream();
+    const stream = await resume.resumableStream('test', () => readable);
+    writer.write('1\n');
+    writer.write('2\n');
+    const stream2 = await resume.resumableStream('test', () => readable, 4);
+    writer.close();
+    const result = await streamToBuffer(stream);
+    const result2 = await streamToBuffer(stream2);
+    expect(result).toEqual('1\n2\n');
+    expect(result2).toEqual('');
+  });
+
+  it('should respect skipCharacters 0', async () => {
+    const { readable, writer } = createTestingStream();
+    const stream = await resume.resumableStream('test', () => readable);
+    writer.write('1\n');
+    writer.write('2\n');
+    const stream2 = await resume.resumableStream('test', () => readable, 0);
+    writer.close();
+    const result = await streamToBuffer(stream);
+    const result2 = await streamToBuffer(stream2);
+    expect(result).toEqual('1\n2\n');
+    expect(result2).toEqual('1\n2\n');
+  });
+
+  // We return the stream because we can afford to hold on to the old streams
+  // Leaving this as a skipped test in case we want to offer a compat mode that
+  // returns null
+  it.skip('should return null if stream is done', async () => {
+    const { readable, writer } = createTestingStream();
+    const stream = await resume.resumableStream('test', () => readable);
+    writer.write('1\n');
+    writer.write('2\n');
+    writer.close();
+
+    const result = await streamToBuffer(stream);
+    expect(
+      await resume.resumableStream('test', () => {
+        throw new Error('Should never be called');
+      }),
+    ).toBeNull();
+    expect(result).toEqual('1\n2\n');
+  });
+
+  it('should support the deconstructed APIs', async () => {
+    const { readable, writer } = createTestingStream();
+    const stream = await resume.createNewResumableStream(
+      'test',
+      () => readable,
+    );
+    const stream2 = await resume.resumeExistingStream('test');
+    writer.write('1\n');
+    writer.write('2\n');
+    writer.close();
+    const result = await streamToBuffer(stream);
+    const result2 = await streamToBuffer(stream2);
+    expect(result).toEqual('1\n2\n');
+    expect(result2).toEqual('1\n2\n');
+  });
+
+  // We return the stream because we can afford to hold on to the old streams
+  // Leaving this as a skipped test in case we want to offer a compat mode that
+  // returns null
+  it.skip('should return null if stream is done explicit APIs', async () => {
+    const { readable, writer } = createTestingStream();
+    const stream = await resume.createNewResumableStream(
+      'test',
+      () => readable,
+    );
+    writer.write('1\n');
+    writer.write('2\n');
+    writer.close();
+
+    const result = await streamToBuffer(stream);
+    expect(await resume.resumeExistingStream('test')).toBeNull();
+    expect(result).toEqual('1\n2\n');
+  });
+});

package/__tests__/src/testing-stream.ts

@@ -0,0 +1,87 @@
+export function createTestingStream() {
+  let controller: ReadableStreamDefaultController<string> | undefined =
+    undefined;
+  const buffer: string[] = [];
+  const readable = new ReadableStream<string>({
+    start(c) {
+      controller = c;
+      if (buffer.length > 0) {
+        for (const chunk of buffer) {
+          controller.enqueue(chunk);
+        }
+      }
+    },
+  });
+
+  const writable = new WritableStream<string>({
+    write(chunk) {
+      if (controller) {
+        controller.enqueue(chunk);
+      }
+      buffer.push(chunk);
+    },
+    close() {
+      controller?.close();
+    },
+    abort(reason) {
+      controller?.error(reason);
+    },
+  });
+
+  return {
+    readable,
+    writer: writable.getWriter(),
+    buffer,
+  };
+}
+
+const readers = new WeakMap<
+  ReadableStream<string>,
+  ReadableStreamDefaultReader<string>
+>();
+
+export async function streamToBuffer(
+  stream: ReadableStream<string> | null | undefined,
+  maxNReads?: number,
+) {
+  if (stream === null) {
+    throw new Error('Stream should not be null');
+  }
+  if (stream === undefined) {
+    throw new Error('Stream should not be undefined');
+  }
+
+  const reader = (
+    readers.has(stream) ? readers.get(stream) : stream.getReader()
+  ) as ReadableStreamDefaultReader<string>;
+  readers.set(stream, reader);
+
+  const buffer: string[] = [];
+  function timeout(ms: number) {
+    return new Promise((_, reject) =>
+      setTimeout(
+        () =>
+          reject(new Error(`Timeout with buffer ${JSON.stringify(buffer)}`)),
+        ms,
+      ),
+    );
+  }
+
+  let i = 0;
+  while (true) {
+    const { done, value } = await (Promise.race([
+      reader.read(),
+      timeout(2000),
+    ]) as Promise<{ done: boolean; value: string }>);
+    if (!done) {
+      buffer.push(value);
+    }
+    if (maxNReads && ++i === maxNReads) {
+      break;
+    }
+    if (done) {
+      break;
+    }
+  }
+  return buffer.join('');
+}

package/backup.ts

@@ -0,0 +1,8 @@
+import { defineConfig } from 'vitest/config';
+
+export default defineConfig({
+  test: {
+    include: ['src/__tests__/**/*.test.ts'],
+    exclude: ['vercel-resumable-stream/**', 'node_modules/**', 'dist/**'],
+  },
+});