@indigoai-us/hq-cloud 6.14.50 → 6.15.1

package/src/sources/get.test.ts

@@ -1,293 +0,0 @@
-/**
- * Unit tests for sources/get.ts.
- */
-
-import { describe, it, expect, vi, beforeEach, afterEach } from "vitest";
-import { Readable } from "node:stream";
-import { GetObjectCommand, type S3Client } from "@aws-sdk/client-s3";
-import { getSource, SourceNotFoundError } from "./get.js";
-import {
-  _setSourcesS3Factory,
-  _resetSourcesS3Factory,
-} from "./internals.js";
-import {
-  setPresignedGetTransportForTesting,
-  type PresignedGetTransport,
-  type PresignTransportClient,
-} from "../object-io.js";
-import type { EntityContext } from "../types.js";
-import { InvalidSourceChannelError } from "../schemas/source-channels.js";
-
-// ---------------------------------------------------------------------------
-// Fixtures
-// ---------------------------------------------------------------------------
-
-const ENTITY: EntityContext = {
-  uid: "cmp_test_001",
-  slug: "test",
-  bucketName: "hq-vault-test",
-  region: "us-east-1",
-  credentials: {
-    accessKeyId: "ASIATEST",
-    secretAccessKey: "secret",
-    sessionToken: "session",
-  },
-  expiresAt: new Date(Date.now() + 3600_000).toISOString(),
-};
-
-const MEETING_MD = `---
-source_id: abc
-source_type: meeting
-title: Q1 Planning
----
-
-Body text line 1.
-Body text line 2.
-`;
-
-const RAW_PAYLOAD = { foo: "bar", n: 42 };
-
-function installStub(objects: Record<string, string>): void {
-  async function send(command: unknown): Promise<unknown> {
-    if (command instanceof GetObjectCommand) {
-      const { Key } = command.input as { Key: string };
-      const content = objects[Key];
-      if (content === undefined) {
-        const err = new Error(`NoSuchKey: ${Key}`);
-        (err as Error & { name: string }).name = "NoSuchKey";
-        throw err;
-      }
-      async function* stream(): AsyncIterable<Uint8Array> {
-        yield Buffer.from(content, "utf-8");
-      }
-      return { Body: stream(), $metadata: {} };
-    }
-    throw new Error(`unhandled command`);
-  }
-  _setSourcesS3Factory(() => ({ send } as unknown as S3Client));
-}
-
-beforeEach(() => _resetSourcesS3Factory());
-afterEach(() => _resetSourcesS3Factory());
-
-// ---------------------------------------------------------------------------
-// Tests
-// ---------------------------------------------------------------------------
-
-describe("getSource", () => {
-  it("happy path: parses frontmatter and body", async () => {
-    installStub({ "sources/meetings/abc.md": MEETING_MD });
-
-    const doc = await getSource({
-      entity: ENTITY,
-      channel: "meeting",
-      sourceId: "abc",
-    });
-
-    expect(doc.key).toBe("sources/meetings/abc.md");
-    expect(doc.frontmatter).toEqual({
-      source_id: "abc",
-      source_type: "meeting",
-      title: "Q1 Planning",
-    });
-    expect(doc.body).toContain("Body text line 1.");
-    expect(doc.body).toContain("Body text line 2.");
-    expect(doc.raw).toBeUndefined();
-  });
-
-  it("throws SourceNotFoundError on NoSuchKey", async () => {
-    installStub({});
-
-    await expect(
-      getSource({ entity: ENTITY, channel: "meeting", sourceId: "missing" }),
-    ).rejects.toBeInstanceOf(SourceNotFoundError);
-
-    await expect(
-      getSource({ entity: ENTITY, channel: "meeting", sourceId: "missing" }),
-    ).rejects.toThrow(/sources\/meetings\/missing\.md/);
-  });
-
-  it("includeRaw:true fetches the .raw.json sibling", async () => {
-    installStub({
-      "sources/meetings/abc.md": MEETING_MD,
-      "sources/meetings/abc.raw.json": JSON.stringify(RAW_PAYLOAD),
-    });
-
-    const doc = await getSource({
-      entity: ENTITY,
-      channel: "meeting",
-      sourceId: "abc",
-      includeRaw: true,
-    });
-
-    expect(doc.raw).toEqual(RAW_PAYLOAD);
-  });
-
-  it("includeRaw:true tolerates missing .raw.json (raw stays undefined)", async () => {
-    installStub({ "sources/meetings/abc.md": MEETING_MD });
-
-    const doc = await getSource({
-      entity: ENTITY,
-      channel: "meeting",
-      sourceId: "abc",
-      includeRaw: true,
-    });
-
-    expect(doc.frontmatter).not.toBeNull();
-    expect(doc.raw).toBeUndefined();
-  });
-
-  it("rejects an invalid channel via assertSourceChannel BEFORE any S3 call", async () => {
-    let called = 0;
-    _setSourcesS3Factory(() => ({
-      send: async () => {
-        called++;
-        return {};
-      },
-    } as unknown as S3Client));
-
-    await expect(
-      getSource({
-        entity: ENTITY,
-        // @ts-expect-error — invalid channel intentional
-        channel: "pigeon",
-        sourceId: "abc",
-      }),
-    ).rejects.toBeInstanceOf(InvalidSourceChannelError);
-    expect(called).toBe(0);
-  });
-
-  it("returns frontmatter:null for malformed (no frontmatter block) documents", async () => {
-    installStub({ "sources/meetings/plain.md": "Just a body, no frontmatter." });
-
-    const doc = await getSource({
-      entity: ENTITY,
-      channel: "meeting",
-      sourceId: "plain",
-    });
-
-    expect(doc.frontmatter).toBeNull();
-    expect(doc.body).toBe("Just a body, no frontmatter.");
-  });
-});
-
-// ---------------------------------------------------------------------------
-// Presigned transport (vault client + company vault) — HQ-59
-// ---------------------------------------------------------------------------
-
-function fetchBackedPresignedGetTransport(): PresignedGetTransport {
-  return {
-    async get(url, headers) {
-      const response = await fetch(url, { method: "GET", headers });
-      const body = response.body ? Readable.fromWeb(response.body) : undefined;
-      return {
-        status: response.status,
-        headers: response.headers,
-        body,
-        destroy: () => body?.destroy(),
-      };
-    },
-  };
-}
-
-/**
- * A presign-capable vault stub + a `fetch` mock keyed by the presigned URL.
- * presign() mints a URL that encodes the key; fetch() returns the object's
- * bytes (404 when absent, so a missing key surfaces as SourceNotFoundError —
- * matching the S3 NoSuchKey path).
- */
-function makePresignVault(objects: Record<string, string>): PresignTransportClient {
-  vi.stubGlobal(
-    "fetch",
-    vi.fn(async (url: string) => {
-      const key = new URL(url).searchParams.get("key") ?? "";
-      const content = objects[key];
-      if (content === undefined) return new Response(null, { status: 404 });
-      return new Response(Buffer.from(content, "utf-8"), { status: 200 });
-    }),
-  );
-  setPresignedGetTransportForTesting(fetchBackedPresignedGetTransport());
-  return {
-    presign: async (input) => ({
-      results: input.keys.map((k) => ({
-        key: k.key,
-        op: k.op ?? input.op ?? "get",
-        url: `https://signed.example/?key=${encodeURIComponent(k.key)}`,
-      })),
-      expiresAt: "2099-01-01T00:00:00.000Z",
-    }),
-    listFiles: async () => ({ objects: [], cursor: null, truncated: false }),
-  };
-}
-
-describe("getSource — presigned transport (vault + cmp_)", () => {
-  afterEach(() => {
-    setPresignedGetTransportForTesting(null);
-    vi.unstubAllGlobals();
-  });
-
-  it("reads via presign and never touches S3", async () => {
-    // S3 factory throws if used — proves the read went through presign.
-    _setSourcesS3Factory(
-      () =>
-        ({
-          send: async () => {
-            throw new Error("S3 must not be used on the presign path");
-          },
-        }) as unknown as S3Client,
-    );
-    const vault = makePresignVault({ "sources/meetings/abc.md": MEETING_MD });
-
-    const doc = await getSource({
-      entity: ENTITY,
-      channel: "meeting",
-      sourceId: "abc",
-      vault,
-    });
-
-    expect(doc.frontmatter).toEqual({
-      source_id: "abc",
-      source_type: "meeting",
-      title: "Q1 Planning",
-    });
-    expect(doc.body).toContain("Body text line 1.");
-  });
-
-  it("maps a presign 404 to SourceNotFoundError", async () => {
-    const vault = makePresignVault({});
-    await expect(
-      getSource({ entity: ENTITY, channel: "meeting", sourceId: "missing", vault }),
-    ).rejects.toBeInstanceOf(SourceNotFoundError);
-  });
-
-  it("includeRaw via presign reads the .raw.json sibling", async () => {
-    const vault = makePresignVault({
-      "sources/meetings/abc.md": MEETING_MD,
-      "sources/meetings/abc.raw.json": JSON.stringify(RAW_PAYLOAD),
-    });
-    const doc = await getSource({
-      entity: ENTITY,
-      channel: "meeting",
-      sourceId: "abc",
-      includeRaw: true,
-      vault,
-    });
-    expect(doc.raw).toEqual(RAW_PAYLOAD);
-  });
-
-  it("personal vault (prs_) ignores the vault client and stays on direct S3", async () => {
-    // S3 stub serves the doc; the presign fetch would 404 if (wrongly) used.
-    installStub({ "sources/meetings/abc.md": MEETING_MD });
-    const vault = makePresignVault({});
-    const personal: EntityContext = { ...ENTITY, uid: "prs_me" };
-
-    const doc = await getSource({
-      entity: personal,
-      channel: "meeting",
-      sourceId: "abc",
-      vault,
-    });
-
-    expect(doc.body).toContain("Body text line 1.");
-  });
-});

package/src/sources/get.ts

@@ -1,66 +0,0 @@
-/**
- * getSource — fetch a single source document (and optionally its `.raw.json`
- * sibling) from an entity bucket and return parsed frontmatter + body.
- */
-
-import { assertSourceChannel, sourcePathSegment } from "../schemas/source-channels.js";
-import { readObjectText, tryReadObjectText } from "./internals.js";
-import { parseMarkdown } from "./parse.js";
-import type { GetSourceOptions, SourceDocument } from "./types.js";
-
-export class SourceNotFoundError extends Error {
-  constructor(public readonly key: string) {
-    super(`Source not found: ${key}`);
-    this.name = "SourceNotFoundError";
-  }
-}
-
-function isNoSuchKey(err: unknown): boolean {
-  if (!err || typeof err !== "object") return false;
-  const name = (err as { name?: unknown }).name;
-  // The transport normalizes a missing object to a `NoSuchKey`-named error on
-  // both the S3 and presigned paths (a presign 404 → NoSuchKey). `NotFound` is
-  // kept for defense in depth (S3 HEAD-style errors).
-  return (
-    name === "NoSuchKey" || name === "NoSuchKeyException" || name === "NotFound"
-  );
-}
-
-export async function getSource(opts: GetSourceOptions): Promise<SourceDocument> {
-  // Validate channel BEFORE any read (acceptance criterion).
-  assertSourceChannel(opts.channel);
-
-  // sources-pipeline writes "meeting" channel to sources/meetings/...; other
-  // channels pass through. See schemas/source-channels.ts:sourcePathSegment.
-  const segment = sourcePathSegment(opts.channel);
-  const key = `sources/${segment}/${opts.sourceId}.md`;
-
-  let body: string;
-  try {
-    body = await readObjectText(opts, key);
-  } catch (err) {
-    if (isNoSuchKey(err)) {
-      throw new SourceNotFoundError(key);
-    }
-    throw err;
-  }
-
-  const parsed = parseMarkdown(body);
-
-  const document: SourceDocument = {
-    key,
-    frontmatter: parsed.frontmatter,
-    body: parsed.body,
-  };
-
-  if (opts.includeRaw) {
-    // Raw sibling is optional; tryReadObjectText returns null when absent.
-    const rawKey = `sources/${segment}/${opts.sourceId}.raw.json`;
-    const rawText = await tryReadObjectText(opts, rawKey);
-    if (rawText !== null) {
-      document.raw = JSON.parse(rawText);
-    }
-  }
-
-  return document;
-}

package/src/sources/internals.ts

@@ -1,198 +0,0 @@
-/**
- * Internal helpers shared between sources/list.ts and sources/get.ts.
- *
- * Two transports back the read surface:
- *   - presigned-URL (HQ-59): when a vault client is supplied AND the entity is a
- *     company vault (cmp_*), reads go through the vault `list`/`presign`
- *     endpoints — the client never talks to S3 directly. This is the path every
- *     company read should take going forward.
- *   - direct S3 (legacy / personal): no vault client, or a personal vault
- *     (prs_*, whose membership-less creds 403 the membership-gated presign
- *     endpoints). Goes through `getS3Client`, whose factory hook lets tests
- *     inject an in-memory stub without vi.mock.
- */
-
-import {
-  S3Client,
-  GetObjectCommand,
-  ListObjectsV2Command,
-} from "@aws-sdk/client-s3";
-import type { EntityContext } from "../types.js";
-import { PresignObjectIO, type PresignTransportClient } from "../object-io.js";
-
-function defaultFactory(ctx: EntityContext): S3Client {
-  if (!ctx.credentials) {
-    // The direct-S3 read path needs STS creds. A credential-less context only
-    // exists on the presign path (HQ-59 company vend-skip), which reads via the
-    // vault list/presign endpoints, not this S3 client — so arriving here with
-    // no creds is a routing bug. Fail loudly rather than 403 opaquely later.
-    throw new Error(
-      `Sources S3 read for ${ctx.uid} requires STS credentials but got a ` +
-        `presign-only context; company presign reads must go through the ` +
-        `vault transport, not getS3Client.`,
-    );
-  }
-  return new S3Client({
-    region: ctx.region,
-    credentials: {
-      accessKeyId: ctx.credentials.accessKeyId,
-      secretAccessKey: ctx.credentials.secretAccessKey,
-      sessionToken: ctx.credentials.sessionToken,
-    },
-  });
-}
-
-let factory: (ctx: EntityContext) => S3Client = defaultFactory;
-
-export function getS3Client(ctx: EntityContext): S3Client {
-  return factory(ctx);
-}
-
-/** Test hook: replace the S3 client factory. */
-export function _setSourcesS3Factory(f: (ctx: EntityContext) => S3Client): void {
-  factory = f;
-}
-
-/** Test hook: restore the default S3 client factory. */
-export function _resetSourcesS3Factory(): void {
-  factory = defaultFactory;
-}
-
-export async function streamToString(body: unknown): Promise<string> {
-  const stream = body as AsyncIterable<Uint8Array>;
-  const chunks: Buffer[] = [];
-  for await (const chunk of stream) {
-    chunks.push(Buffer.from(chunk));
-  }
-  return Buffer.concat(chunks).toString("utf-8");
-}
-
-// ---------------------------------------------------------------------------
-// Transport seam (presign for cmp_, direct-S3 otherwise)
-// ---------------------------------------------------------------------------
-
-/** Common shape of every reader's options: an entity + an optional vault client. */
-export interface ReaderTransportOpts {
-  entity: EntityContext;
-  /**
-   * Presign-capable vault client. When supplied and the entity is a company
-   * vault (cmp_*), reads use the presigned-URL transport instead of direct S3.
-   */
-  vault?: PresignTransportClient;
-}
-
-/** Lightweight listed object (transport-agnostic). */
-export interface ListedKey {
-  key: string;
-  size: number;
-  lastModified: Date;
-}
-
-/** True when this read should use the presigned-URL transport. */
-export function usePresign(opts: ReaderTransportOpts): boolean {
-  return opts.vault != null && opts.entity.uid.startsWith("cmp_");
-}
-
-/** A not-found error normalized to the S3 `NoSuchKey` shape callers test for. */
-function noSuchKeyError(key: string): Error {
-  return Object.assign(new Error(`No such key: ${key}`), { name: "NoSuchKey" });
-}
-
-function isNotFound(err: unknown): boolean {
-  if (!err || typeof err !== "object") return false;
-  const name = (err as { name?: unknown }).name;
-  return (
-    name === "NoSuchKey" || name === "NoSuchKeyException" || name === "NotFound"
-  );
-}
-
-/**
- * Read an object's UTF-8 text. Throws a `NoSuchKey`-named error when the object
- * is absent (both transports), so callers' existing not-found checks work
- * unchanged.
- */
-export async function readObjectText(
-  opts: ReaderTransportOpts,
-  key: string,
-): Promise<string> {
-  if (usePresign(opts)) {
-    const io = new PresignObjectIO(opts.vault!, opts.entity.uid);
-    try {
-      const { body } = await io.getObject(key);
-      return body.toString("utf-8");
-    } catch (err) {
-      if (isNotFound(err)) throw noSuchKeyError(key);
-      throw err;
-    }
-  }
-  const client = getS3Client(opts.entity);
-  const res = await client.send(
-    new GetObjectCommand({ Bucket: opts.entity.bucketName, Key: key }),
-  );
-  return streamToString(res.Body);
-}
-
-/** Like {@link readObjectText} but returns null when the object is absent. */
-export async function tryReadObjectText(
-  opts: ReaderTransportOpts,
-  key: string,
-): Promise<string | null> {
-  try {
-    return await readObjectText(opts, key);
-  } catch (err) {
-    if (isNotFound(err)) return null;
-    throw err;
-  }
-}
-
-/**
- * List one page of objects under `prefix`.
- *
- * Under the presigned transport the vault `list` endpoint is ACL-filtered and
- * controls page size server-side, so `limit` is advisory — paginate via the
- * returned `nextToken` (the opaque vault cursor) until it is undefined. Under
- * direct S3, `limit` maps to `MaxKeys`.
- */
-export async function listObjects(
-  opts: ReaderTransportOpts,
-  prefix: string,
-  page: { limit?: number; continuationToken?: string },
-): Promise<{ contents: ListedKey[]; nextToken?: string }> {
-  if (usePresign(opts)) {
-    const io = new PresignObjectIO(opts.vault!, opts.entity.uid);
-    const res = await io.listObjects({
-      prefix,
-      continuationToken: page.continuationToken,
-    });
-    return {
-      contents: res.objects.map((o) => ({
-        key: o.key,
-        size: o.size,
-        lastModified: o.lastModified,
-      })),
-      nextToken: res.nextContinuationToken,
-    };
-  }
-  const client = getS3Client(opts.entity);
-  const res = await client.send(
-    new ListObjectsV2Command({
-      Bucket: opts.entity.bucketName,
-      Prefix: prefix,
-      MaxKeys: page.limit,
-      ContinuationToken: page.continuationToken,
-    }),
-  );
-  const contents: ListedKey[] = [];
-  for (const obj of res.Contents ?? []) {
-    if (!obj.Key) continue;
-    contents.push({
-      key: obj.Key,
-      size: obj.Size ?? 0,
-      lastModified: obj.LastModified ?? new Date(0),
-    });
-  }
-  return {
-    contents,
-    nextToken: res.IsTruncated ? res.NextContinuationToken : undefined,
-  };
-}