@remit/backend 0.0.29 → 0.0.31

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@remit/backend",
-  "version": "0.0.29",
+  "version": "0.0.31",
   "description": "Remit Mail Inspector API backend",
   "license": "MIT",
   "author": "",

package/src/handlers/system-update.test.ts

@@ -9,9 +9,7 @@ import {
 import { join } from "node:path";
 import { after, afterEach, before, beforeEach, describe, it } from "node:test";
 import { fileURLToPath } from "node:url";
-import { createInstanceOwnerStore } from "@remit/auth-service";
 import type { APIGatewayProxyEvent } from "aws-lambda";
-import Database from "better-sqlite3";
 import type { Context } from "openapi-backend";
 import { SystemOperations } from "./system-update.js";
 
@@ -23,20 +21,9 @@ const tmpRoot = join(
 	"system-update",
 );
 
-const readMigrationSql = (relativePath: string): string =>
-	readFileSync(fileURLToPath(new URL(relativePath, import.meta.url)), "utf8");
-
-const authMigrationSql = readMigrationSql(
-	"../../../../deploy/vps/migrations-sqlite/auth/0000_rainy_iron_monger.sql",
-);
-const metaMigrationSql = readMigrationSql(
-	"../../../../deploy/vps/migrations-sqlite/meta/0000_demonic_lilith.sql",
-);
-
-const OWNER = "the-owner";
+const USER = "any-signed-in-user";
 const MANIFEST_URL = "https://updates.example.com/stable.json";
 
-let dbPath: string;
 let controlDir: string;
 
 const okState = {
@@ -57,29 +44,8 @@ const writeState = (state: unknown): void => {
 	writeFileSync(join(controlDir, "state.json"), JSON.stringify(state));
 };
 
-before(async () => {
+before(() => {
 	mkdirSync(tmpRoot, { recursive: true });
-	dbPath = join(mkdtempSync(join(tmpRoot, "db-")), "app.db");
-	const sqlite = new Database(dbPath);
-	for (const sql of [authMigrationSql, metaMigrationSql]) {
-		for (const statement of sql.split("--> statement-breakpoint")) {
-			sqlite.exec(statement);
-		}
-	}
-	sqlite
-		.prepare(
-			"INSERT INTO auth_user (id, name, email, email_verified, created_at, updated_at) VALUES (?, ?, ?, 0, ?, ?)",
-		)
-		.run(OWNER, OWNER, "owner@example.com", 1000, 1000);
-	sqlite.close();
-	process.env.DATA_BACKEND = "sqlite";
-	process.env.SQLITE_DB_PATH = dbPath;
-	const store = await createInstanceOwnerStore({
-		provider: "sqlite",
-		connectionString: dbPath,
-	});
-	await store.claimIfUnclaimed(OWNER);
-	await store.close();
 });
 
 after(() => {
@@ -135,27 +101,37 @@ describe("GET /system/update", () => {
 	it("returns 404 when no manifest URL is configured", async () => {
 		delete process.env.REMIT_UPDATE_MANIFEST_URL;
 
-		const result = await getUpdate(buildEvent(OWNER));
+		const result = await getUpdate(buildEvent(USER));
 
 		assert.ok(hasStatus(result, 404));
 	});
 
-	it("returns 403 for a non-owner", async () => {
-		const result = await getUpdate(buildEvent("someone-else"));
+	it("returns 401 when the caller is not authenticated", async () => {
+		writeState(okState);
+
+		const result = await getUpdate(buildEvent());
+
+		assert.ok(hasStatus(result, 401));
+	});
+
+	it("returns 200 for any authenticated caller", async () => {
+		writeState(okState);
+
+		const result = await getUpdate(buildEvent("another-signed-in-user"));
 
-		assert.ok(hasStatus(result, 403));
+		assert.deepEqual(result, okState);
 	});
 
 	it("returns 200 with the state file verbatim", async () => {
 		writeState(okState);
 
-		const result = await getUpdate(buildEvent(OWNER));
+		const result = await getUpdate(buildEvent(USER));
 
 		assert.deepEqual(result, okState);
 	});
 
 	it("returns a disabled, empty resource when no state file exists", async () => {
-		const result = await getUpdate(buildEvent(OWNER));
+		const result = await getUpdate(buildEvent(USER));
 
 		assert.deepEqual(result, {
 			currentVersion: "v1.4.1",
@@ -186,7 +162,7 @@ describe("GET /system/update", () => {
 			run,
 		});
 
-		const result = (await getUpdate(buildEvent(OWNER))) as {
+		const result = (await getUpdate(buildEvent(USER))) as {
 			check: { status: string };
 			run: unknown;
 		};
@@ -203,7 +179,7 @@ describe("GET /system/update", () => {
 			run: null,
 		});
 
-		const result = (await getUpdate(buildEvent(OWNER))) as {
+		const result = (await getUpdate(buildEvent(USER))) as {
 			currentSchemaVersion?: number;
 			check: { schemaVersion?: number };
 		};
@@ -215,7 +191,7 @@ describe("GET /system/update", () => {
 	it("omits both schema-version fields when the state file predates them", async () => {
 		writeState(okState);
 
-		const result = (await getUpdate(buildEvent(OWNER))) as {
+		const result = (await getUpdate(buildEvent(USER))) as {
 			currentSchemaVersion?: number;
 			check: { schemaVersion?: number };
 		};
@@ -232,7 +208,7 @@ describe("GET /system/update", () => {
 			run: null,
 		});
 
-		const result = (await getUpdate(buildEvent(OWNER))) as {
+		const result = (await getUpdate(buildEvent(USER))) as {
 			currentSchemaVersion?: number;
 			check: { schemaVersion?: number };
 		};
@@ -248,7 +224,7 @@ describe("GET /system/update", () => {
 			run: null,
 		});
 
-		const result = (await getUpdate(buildEvent(OWNER))) as {
+		const result = (await getUpdate(buildEvent(USER))) as {
 			currentSchemaVersion?: number;
 			check: { schemaVersion?: number };
 		};
@@ -262,17 +238,34 @@ describe("POST /system/update", () => {
 	it("returns 404 when no manifest URL is configured", async () => {
 		delete process.env.REMIT_UPDATE_MANIFEST_URL;
 
-		const result = await applyUpdate("v1.5.0", buildEvent(OWNER));
+		const result = await applyUpdate("v1.5.0", buildEvent(USER));
 
 		assert.ok(hasStatus(result, 404));
 	});
 
-	it("returns 403 for a non-owner", async () => {
+	it("returns 401 without writing request.json when the caller is not authenticated", async () => {
+		writeState(okState);
+
+		const result = await applyUpdate("v1.5.0", buildEvent());
+
+		assert.ok(hasStatus(result, 401));
+		assert.throws(() => readFileSync(join(controlDir, "request.json"), "utf8"));
+	});
+
+	it("returns 202 for any authenticated caller and records that identity", async () => {
 		writeState(okState);
 
-		const result = await applyUpdate("v1.5.0", buildEvent("someone-else"));
+		const result = (await applyUpdate(
+			"v1.5.0",
+			buildEvent("another-signed-in-user"),
+		)) as { statusCode: number };
+
+		assert.equal(result.statusCode, 202);
 
-		assert.ok(hasStatus(result, 403));
+		const request = JSON.parse(
+			readFileSync(join(controlDir, "request.json"), "utf8"),
+		);
+		assert.equal(request.requestedBy, "another-signed-in-user");
 	});
 
 	it("returns 409 when a run is already in flight", async () => {
@@ -291,7 +284,7 @@ describe("POST /system/update", () => {
 			},
 		});
 
-		await assert.rejects(applyUpdate("v1.5.0", buildEvent(OWNER)), {
+		await assert.rejects(applyUpdate("v1.5.0", buildEvent(USER)), {
 			statusCode: 409,
 		});
 	});
@@ -299,13 +292,13 @@ describe("POST /system/update", () => {
 	it("returns 422 when the target is not the checked version", async () => {
 		writeState(okState);
 
-		await assert.rejects(applyUpdate("v9.9.9", buildEvent(OWNER)), {
+		await assert.rejects(applyUpdate("v9.9.9", buildEvent(USER)), {
 			statusCode: 422,
 		});
 	});
 
 	it("returns 422 when no check has ever succeeded", async () => {
-		await assert.rejects(applyUpdate("v1.5.0", buildEvent(OWNER)), {
+		await assert.rejects(applyUpdate("v1.5.0", buildEvent(USER)), {
 			statusCode: 422,
 		});
 	});
@@ -313,7 +306,7 @@ describe("POST /system/update", () => {
 	it("returns 202 with a run block carrying a fresh runId", async () => {
 		writeState(okState);
 
-		const result = (await applyUpdate("v1.5.0", buildEvent(OWNER))) as {
+		const result = (await applyUpdate("v1.5.0", buildEvent(USER))) as {
 			statusCode: number;
 			body: { run: { runId: string; targetVersion: string; outcome: null } };
 		};
@@ -327,7 +320,7 @@ describe("POST /system/update", () => {
 	it("writes request.json with exactly the four seam fields and nothing else", async () => {
 		writeState(okState);
 
-		await applyUpdate("v1.5.0", buildEvent(OWNER));
+		await applyUpdate("v1.5.0", buildEvent(USER));
 
 		const request = JSON.parse(
 			readFileSync(join(controlDir, "request.json"), "utf8"),
@@ -340,7 +333,7 @@ describe("POST /system/update", () => {
 			"targetVersion",
 		]);
 		assert.equal(request.targetVersion, "v1.5.0");
-		assert.equal(request.requestedBy, OWNER);
+		assert.equal(request.requestedBy, USER);
 		for (const forbidden of ["registry", "image", "digest"]) {
 			assert.ok(!(forbidden in request));
 		}

package/src/handlers/system-update.ts

@@ -10,7 +10,6 @@ import type { APIGatewayProxyEvent, APIGatewayProxyResult } from "aws-lambda";
 import type { Context } from "openapi-backend";
 import { getSubFromEvent } from "../auth.js";
 import type { OperationHandler, SystemOperationIds } from "../types.js";
-import { requireInstanceOwner } from "./owner-guard.js";
 
 class TargetVersionMismatchError extends HTTPError {
 	name = "TargetVersionMismatchError";
@@ -33,11 +32,17 @@ const notFound = (): APIGatewayProxyResult => ({
 	body: JSON.stringify({ message: "Not found" }),
 });
 
+const unauthorized = (): APIGatewayProxyResult => ({
+	statusCode: 401,
+	headers: { "Content-Type": "application/json" },
+	body: JSON.stringify({ message: "Unauthorized" }),
+});
+
 /**
  * The self-update seam is off unless a manifest URL is configured (RFC 037 D8),
  * which the hosted deployment leaves unset. Both endpoints answer 404 in that
- * case — before the owner guard, so a probe cannot tell an off surface from a
- * forbidden one.
+ * case. Where it is on, any authenticated caller may use it — the account list
+ * is the trust boundary of a self-hosted instance.
  */
 const guardManifestConfigured = (): APIGatewayProxyResult | null =>
 	manifestUrl() ? null : notFound();
@@ -140,8 +145,7 @@ export const SystemOperations: Record<
 		if (offSurface) return offSurface;
 
 		const event = args[0] as APIGatewayProxyEvent;
-		const forbidden = await requireInstanceOwner(event);
-		if (forbidden) return forbidden;
+		if (!getSubFromEvent(event)) return unauthorized();
 
 		return readState() ?? emptyResource();
 	},
@@ -154,8 +158,8 @@ export const SystemOperations: Record<
 		if (offSurface) return offSurface;
 
 		const event = args[0] as APIGatewayProxyEvent;
-		const forbidden = await requireInstanceOwner(event);
-		if (forbidden) return forbidden;
+		const sub = getSubFromEvent(event);
+		if (!sub) return unauthorized();
 
 		const { targetVersion } = context.request.requestBody as {
 			targetVersion: string;
@@ -178,7 +182,7 @@ export const SystemOperations: Record<
 			runId,
 			targetVersion,
 			requestedAt,
-			requestedBy: getSubFromEvent(event) ?? "",
+			requestedBy: sub,
 		});
 
 		return {

package/src/service/organize.test.ts

@@ -213,7 +213,14 @@ const candidate = (
 	over: Partial<OrganizeCandidate["message"]> = {},
 ): OrganizeCandidate => ({
 	messageId,
-	message: { from: "", fromName: "", subject: "", text: "", ...over },
+	message: {
+		from: "",
+		fromName: "",
+		subject: "",
+		text: "",
+		listId: "",
+		...over,
+	},
 });
 
 describe("matchOrganize", () => {
@@ -597,3 +604,115 @@ describe("back-apply pipeline (matchOrganize -> applyOrganize)", () => {
 		);
 	});
 });
+
+describe("matchOrganize with ListId and FromDomain clauses", () => {
+	const senderChunk = (
+		messageId: string,
+		fromEmail: string,
+		over: Partial<ChunkMetadata> = {},
+	): VectorRecord => ({
+		chunkId: `${messageId}#sender`,
+		vector: ANCHOR_VECTOR,
+		metadata: metadata({
+			messageId,
+			chunkType: "sender",
+			textPreview: fromEmail,
+			...over,
+		}),
+	});
+
+	it("matches a ListId clause exactly off the vector-free projection", async () => {
+		const deps = vectorlessDeps([
+			candidate("msg-1", { listId: "weekly.news.example.com" }),
+			candidate("msg-2", { listId: "news.example.com" }),
+			candidate("msg-3", {}),
+		]);
+
+		const { messageIds } = await matchOrganize(deps, ACCOUNT_CONFIG_ID, {
+			...predicate(),
+			anchorMessageId: "None",
+			literalClauses: [{ field: "ListId", value: "weekly.news.example.com" }],
+		});
+
+		assert.deepEqual(messageIds, ["msg-1"]);
+	});
+
+	it("matches a FromDomain clause public-suffix aware off the vector-free projection", async () => {
+		const deps = vectorlessDeps([
+			candidate("msg-1", { from: "notifications@github.com" }),
+			candidate("msg-2", { from: "ci@sub.github.com" }),
+			candidate("msg-3", { from: "attacker@github.com.evil.example" }),
+		]);
+
+		const { messageIds } = await matchOrganize(deps, ACCOUNT_CONFIG_ID, {
+			...predicate(),
+			anchorMessageId: "None",
+			literalClauses: [{ field: "FromDomain", value: "github.com" }],
+		});
+
+		assert.deepEqual(messageIds, ["msg-1", "msg-2"]);
+	});
+
+	it("round-trips ListId and FromDomain through the semantic-arm chunk projection", async () => {
+		const store = createMemoryVectorStore();
+		await store.upsert([
+			senderChunk("msg-1", "ci@github.com", {
+				listId: "actions.github.com",
+			}),
+			bodyChunk("msg-1", ANCHOR_VECTOR, { listId: "actions.github.com" }),
+			senderChunk("msg-2", "hi@othersender.example", {
+				listId: "other.list.example",
+			}),
+			bodyChunk("msg-2", ANCHOR_VECTOR, { listId: "other.list.example" }),
+		]);
+
+		const byListId = await matchOrganize(matchDeps(store), ACCOUNT_CONFIG_ID, {
+			...predicate(),
+			literalClauses: [{ field: "ListId", value: "actions.github.com" }],
+		});
+		assert.deepEqual(byListId.messageIds, ["msg-1"]);
+
+		const byFromDomain = await matchOrganize(
+			matchDeps(store),
+			ACCOUNT_CONFIG_ID,
+			{
+				...predicate(),
+				literalClauses: [{ field: "FromDomain", value: "github.com" }],
+			},
+		);
+		assert.deepEqual(byFromDomain.messageIds, ["msg-1"]);
+	});
+
+	it("applies a ListId predicate to exactly the previewed set (preview == apply)", async () => {
+		const deps = vectorlessDeps([
+			candidate("msg-1", { listId: "weekly.news.example.com" }),
+			candidate("msg-2", { listId: "weekly.news.example.com" }),
+			candidate("msg-3", { listId: "other.example.com" }),
+		]);
+		const p = predicate({
+			anchorMessageId: "None",
+			actionLabelId: "lbl-list",
+			literalClauses: [{ field: "ListId", value: "weekly.news.example.com" }],
+		});
+
+		const previewed = await matchOrganize(deps, ACCOUNT_CONFIG_ID, p);
+		const applied = await matchOrganize(deps, ACCOUNT_CONFIG_ID, p);
+		assert.deepEqual(previewed, applied);
+		assert.deepEqual(previewed.messageIds, ["msg-1", "msg-2"]);
+
+		const tracked = trackingClient();
+		const result = await applyOrganize(
+			{ client: tracked.client },
+			ACCOUNT_CONFIG_ID,
+			applied.messageIds,
+			p,
+		);
+
+		assert.equal(result.applied, 2);
+		assert.equal(result.failed, 0);
+		assert.deepEqual(tracked.labeled.map((row) => row.messageId).sort(), [
+			"msg-1",
+			"msg-2",
+		]);
+	});
+});

package/src/service/organize.ts

@@ -131,11 +131,13 @@ const filterMessageFromChunks = (
 	let subject = "";
 	let fromName = "";
 	let from = "";
+	let listId = "";
 	const textParts: string[] = [];
 	for (const record of records) {
 		const meta = record.metadata;
 		if (meta.subject && !subject) subject = meta.subject;
 		if (meta.fromName && !fromName) fromName = meta.fromName;
+		if (meta.listId && !listId) listId = meta.listId;
 		if (meta.chunkType === "sender" && meta.textPreview && !from) {
 			from = meta.textPreview;
 		}
@@ -146,7 +148,7 @@ const filterMessageFromChunks = (
 			textParts.push(meta.textPreview);
 		}
 	}
-	return { from, fromName, subject, text: textParts.join("\n") };
+	return { from, fromName, subject, listId, text: textParts.join("\n") };
 };
 
 /**
@@ -355,8 +357,10 @@ const buildSemanticFromEnv = (): OrganizeSemanticDeps => {
  * deployment that ships no vector pipeline. Deduped by message id — the same
  * mail filed in two folders is one candidate.
  *
- * `From`/`Subject` come from the row verbatim (full fidelity). `text` (the body)
- * is left empty: the thread row carries only a truncated `snippet`, and matching
+ * `From`/`Subject`/`listId` come from the row verbatim (full fidelity), so a
+ * `From`, `Subject`, `FromDomain` or `ListId` clause matches here exactly as it
+ * does at index time. `text` (the body) is left empty: the thread row carries
+ * only a truncated `snippet`, and matching
  * a body-content clause against a preview would silently diverge from the live
  * index-time filter's full-body match. Body-content (`HasWords`) clauses are
  * rejected before this is read (see {@link assertNoBodyContentClause}), so the
@@ -384,6 +388,7 @@ const listAccountFilterMessagesFromClient =
 						fromName: row.fromName ?? "",
 						subject: row.subject ?? "",
 						text: "",
+						listId: row.listId ?? "",
 					},
 				});
 				if (candidates.length >= limit) return candidates;

package/src/handlers/owner-guard.test.ts

@@ -1,89 +0,0 @@
-import assert from "node:assert/strict";
-import { mkdirSync, mkdtempSync, readFileSync, rmSync } from "node:fs";
-import { join } from "node:path";
-import { after, afterEach, before, describe, it } from "node:test";
-import { fileURLToPath } from "node:url";
-import { createInstanceOwnerStore } from "@remit/auth-service";
-import type { APIGatewayProxyEvent } from "aws-lambda";
-import Database from "better-sqlite3";
-import { requireInstanceOwner } from "./owner-guard.js";
-
-const tmpRoot = join(
-	fileURLToPath(new URL(".", import.meta.url)),
-	"..",
-	"..",
-	".tmp",
-	"owner-guard",
-);
-
-const readMigrationSql = (relativePath: string): string =>
-	readFileSync(fileURLToPath(new URL(relativePath, import.meta.url)), "utf8");
-
-const authMigrationSql = readMigrationSql(
-	"../../../../deploy/vps/migrations-sqlite/auth/0000_rainy_iron_monger.sql",
-);
-const metaMigrationSql = readMigrationSql(
-	"../../../../deploy/vps/migrations-sqlite/meta/0000_demonic_lilith.sql",
-);
-
-let dbPath: string;
-
-before(() => {
-	mkdirSync(tmpRoot, { recursive: true });
-	dbPath = join(mkdtempSync(join(tmpRoot, "db-")), "app.db");
-	const sqlite = new Database(dbPath);
-	for (const sql of [authMigrationSql, metaMigrationSql]) {
-		for (const statement of sql.split("--> statement-breakpoint")) {
-			sqlite.exec(statement);
-		}
-	}
-	sqlite
-		.prepare(
-			"INSERT INTO auth_user (id, name, email, email_verified, created_at, updated_at) VALUES (?, ?, ?, 0, ?, ?)",
-		)
-		.run("the-owner", "the-owner", "the-owner@example.com", 1000, 1000);
-	sqlite.close();
-	process.env.DATA_BACKEND = "sqlite";
-	process.env.SQLITE_DB_PATH = dbPath;
-});
-
-after(() => {
-	rmSync(tmpRoot, { recursive: true, force: true });
-});
-
-afterEach(() => {
-	delete process.env.REMIT_OWNER_EMAIL;
-});
-
-const buildEvent = (sub?: string): APIGatewayProxyEvent =>
-	({
-		headers: {},
-		requestContext: sub ? { authorizer: { claims: { sub } } } : {},
-	}) as unknown as APIGatewayProxyEvent;
-
-describe("requireInstanceOwner", () => {
-	it("rejects a request with no authenticated caller", async () => {
-		const result = await requireInstanceOwner(buildEvent());
-
-		assert.equal(result?.statusCode, 403);
-	});
-
-	it("rejects a caller who has not claimed ownership", async () => {
-		const result = await requireInstanceOwner(buildEvent("not-the-owner"));
-
-		assert.equal(result?.statusCode, 403);
-	});
-
-	it("lets the instance owner through", async () => {
-		const store = await createInstanceOwnerStore({
-			provider: "sqlite",
-			connectionString: dbPath,
-		});
-		await store.claimIfUnclaimed("the-owner");
-		await store.close();
-
-		const result = await requireInstanceOwner(buildEvent("the-owner"));
-
-		assert.equal(result, null);
-	});
-});

package/src/handlers/owner-guard.ts

@@ -1,25 +0,0 @@
-import { isInstanceOwner } from "@remit/auth-service";
-import type { APIGatewayProxyEvent, APIGatewayProxyResult } from "aws-lambda";
-import { getSubFromEvent } from "../auth.js";
-
-const forbidden = (message: string): APIGatewayProxyResult => ({
-	statusCode: 403,
-	headers: { "Content-Type": "application/json" },
-	body: JSON.stringify({ message }),
-});
-
-/**
- * Gate a request to the standalone instance owner (RFC 037 D8) — the first
- * account to register, or whoever `REMIT_OWNER_EMAIL` names. Returns `null` to
- * let the request proceed, or a 403 `APIGatewayProxyResult` otherwise. Callers
- * wire this in ahead of the handler; it does not run the handler itself.
- */
-export const requireInstanceOwner = async (
-	event: APIGatewayProxyEvent,
-): Promise<APIGatewayProxyResult | null> => {
-	const sub = getSubFromEvent(event);
-	if (!sub) return forbidden("Instance owner required");
-	if (!(await isInstanceOwner(sub)))
-		return forbidden("Instance owner required");
-	return null;
-};