@de-otio/trellis 0.4.0

package/src/lambda/e2e-sweeper.ts

@@ -0,0 +1,140 @@
+/**
+ * E2E Test Data Sweeper (User-Scoped)
+ *
+ * Safety net for leaked test data. Runs hourly.
+ *
+ * Instead of querying the database directly, this sweeper:
+ * 1. Lists Cognito users with __e2e_ email prefix older than 2 hours
+ * 2. Queues their deletion via the delete-account SQS queue
+ *    (same pipeline used by GDPR account deletions)
+ * 3. Deletes the Cognito user
+ *
+ * The delete-account-worker handles the actual cleanup:
+ * DB cascade (deleteUserData) → S3 media → Cognito identity
+ *
+ * This approach eliminates the risk of a WHERE clause bug deleting real user data.
+ */
+
+import {
+  CognitoIdentityProviderClient,
+  ListUsersCommand,
+  AdminDeleteUserCommand,
+} from "@aws-sdk/client-cognito-identity-provider";
+import { SQSClient, SendMessageCommand } from "@aws-sdk/client-sqs";
+import {
+  CloudWatchClient,
+  PutMetricDataCommand,
+} from "@aws-sdk/client-cloudwatch";
+import { DynamoDBClient, PutItemCommand } from "@aws-sdk/client-dynamodb";
+import { marshall } from "@aws-sdk/util-dynamodb";
+
+const region = process.env.AWS_REGION || "eu-central-1";
+const stage = process.env.STAGE || "dev";
+const userPoolId = process.env.COGNITO_USER_POOL_ID!;
+const deleteQueueUrl = process.env.DELETE_ACCOUNT_QUEUE_URL!;
+const dynamoTable = process.env.DYNAMODB_TABLE!;
+
+const cognito = new CognitoIdentityProviderClient({ region });
+const sqs = new SQSClient({ region });
+const cloudwatch = new CloudWatchClient({ region });
+const dynamo = new DynamoDBClient({ region });
+
+const STALE_THRESHOLD_HOURS = 2;
+const E2E_PREFIX = "__e2e_";
+const MAX_PAGES = 20;
+
+export const handler = async (): Promise<void> => {
+  const cutoff = new Date(Date.now() - STALE_THRESHOLD_HOURS * 60 * 60 * 1000);
+  const now = Math.floor(Date.now() / 1000);
+
+  // Acquire distributed lock
+  try {
+    await dynamo.send(new PutItemCommand({
+      TableName: dynamoTable,
+      Item: marshall({ pk: "cron:e2e-sweeper", sk: "lock", ttl: now + 300, lockedAt: now }),
+      ConditionExpression: "attribute_not_exists(pk) OR #ttl < :now",
+      ExpressionAttributeNames: { "#ttl": "ttl" },
+      ExpressionAttributeValues: marshall({ ":now": now }),
+    }));
+  } catch {
+    console.log(JSON.stringify({ level: "info", msg: "E2E sweeper already running, skipping" }));
+    return;
+  }
+
+  console.log(JSON.stringify({ level: "info", msg: "E2E sweeper started", cutoff: cutoff.toISOString() }));
+
+  let totalQueued = 0;
+
+  // Step 1: List stale __e2e_ Cognito users
+  try {
+    let paginationToken: string | undefined;
+    let pages = 0;
+
+    do {
+      const res = await cognito.send(new ListUsersCommand({
+        UserPoolId: userPoolId,
+        Filter: `email ^= "${E2E_PREFIX}"`,
+        Limit: 60,
+        PaginationToken: paginationToken,
+      }));
+
+      for (const user of res.Users || []) {
+        if (!user.UserCreateDate || user.UserCreateDate >= cutoff) continue;
+        if (!user.Username) continue;
+
+        const email = user.Attributes?.find(a => a.Name === "email")?.Value || user.Username;
+        const sub = user.Attributes?.find(a => a.Name === "sub")?.Value;
+
+        // Step 2: Queue database + S3 deletion via delete-account worker
+        if (sub) {
+          try {
+            await sqs.send(new SendMessageCommand({
+              QueueUrl: deleteQueueUrl,
+              MessageBody: JSON.stringify({ userId: sub }),
+            }));
+            totalQueued++;
+            console.log(JSON.stringify({ level: "info", msg: `Queued deletion for ${email}`, userId: sub }));
+          } catch (err) {
+            console.warn(JSON.stringify({ level: "warn", msg: `Failed to queue deletion for ${email}`, error: String(err) }));
+          }
+        }
+
+        // Step 3: Delete Cognito user immediately
+        // (worker also tries Cognito deletion, but we do it here too
+        //  since the worker may process the message after a delay)
+        try {
+          await cognito.send(new AdminDeleteUserCommand({
+            UserPoolId: userPoolId,
+            Username: user.Username,
+          }));
+          console.log(JSON.stringify({ level: "info", msg: `Deleted Cognito user ${email}` }));
+        } catch (err) {
+          console.warn(JSON.stringify({ level: "warn", msg: `Cognito delete failed for ${email}`, error: String(err) }));
+        }
+      }
+
+      paginationToken = res.PaginationToken;
+      pages++;
+    } while (paginationToken && pages < MAX_PAGES);
+
+  } catch (err) {
+    console.error(JSON.stringify({ level: "error", msg: "Failed to list Cognito users", error: String(err) }));
+  }
+
+  // Step 4: Emit metric
+  try {
+    await cloudwatch.send(new PutMetricDataCommand({
+      Namespace: "Trellis/E2E",
+      MetricData: [{
+        MetricName: "E2eLeakedRecords",
+        Value: totalQueued,
+        Unit: "Count",
+        Dimensions: [{ Name: "Stage", Value: stage }],
+      }],
+    }));
+  } catch (err) {
+    console.error(JSON.stringify({ level: "error", msg: "Failed to emit metric", error: String(err) }));
+  }
+
+  console.log(JSON.stringify({ level: "info", msg: "E2E sweeper complete", totalQueued }));
+};

package/src/lambda/federation-outbox-worker.ts

@@ -0,0 +1,8 @@
+import type { SQSHandler } from "aws-lambda";
+
+export const handler: SQSHandler = async (event) => {
+  for (const record of event.Records) {
+    console.log(JSON.stringify({ level: "info", msg: "Processing record", messageId: record.messageId }));
+    // TODO: implement
+  }
+};

package/src/lambda/followers-events-worker.ts

@@ -0,0 +1,8 @@
+import type { SQSHandler } from "aws-lambda";
+
+export const handler: SQSHandler = async (event) => {
+  for (const record of event.Records) {
+    console.log(JSON.stringify({ level: "info", msg: "Processing record", messageId: record.messageId }));
+    // TODO: implement
+  }
+};

package/src/lambda/hourly-cron.ts

@@ -0,0 +1,103 @@
+import { DynamoDBClient, PutItemCommand } from "@aws-sdk/client-dynamodb";
+import { SecretsManagerClient, GetSecretValueCommand } from "@aws-sdk/client-secrets-manager";
+import { marshall } from "@aws-sdk/util-dynamodb";
+import { PrismaClient } from "@prisma/client";
+
+const dynamo = new DynamoDBClient({ region: process.env.AWS_REGION });
+const secretsClient = new SecretsManagerClient({ region: process.env.AWS_REGION });
+const TABLE = process.env.DYNAMODB_TABLE!;
+
+let prisma: PrismaClient | null = null;
+
+async function getPrisma(): Promise<PrismaClient> {
+  if (prisma) return prisma;
+  const secret = await secretsClient.send(new GetSecretValueCommand({ SecretId: process.env.DB_SECRET_ARN! }));
+  const { username, password, host, port, dbname } = JSON.parse(secret.SecretString!);
+  prisma = new PrismaClient({
+    datasources: { db: { url: `postgresql://${username}:${encodeURIComponent(password)}@${host}:${port}/${dbname}?connection_limit=1` } },
+  });
+  return prisma;
+}
+
+export const handler = async (): Promise<void> => {
+  const now = Math.floor(Date.now() / 1000);
+
+  // Acquire cron lock (prevent overlapping executions)
+  try {
+    await dynamo.send(new PutItemCommand({
+      TableName: TABLE,
+      Item: marshall({
+        pk: "cron:hourly",
+        sk: "lock",
+        ttl: now + 3600, // 1 hour TTL
+        lockedAt: now,
+      }),
+      ConditionExpression: "attribute_not_exists(pk) OR #ttl < :now",
+      ExpressionAttributeNames: { "#ttl": "ttl" },
+      ExpressionAttributeValues: marshall({ ":now": now }),
+    }));
+  } catch {
+    console.log(JSON.stringify({ level: "info", msg: "Hourly cron already running, skipping" }));
+    return;
+  }
+
+  console.log(JSON.stringify({ level: "info", msg: "Hourly cron started" }));
+
+  const db = await getPrisma();
+
+  // 1. Clean up stale PENDING/FAILED media records (older than 1 hour)
+  try {
+    const oneHourAgo = new Date(Date.now() - 3600000);
+    const staleMedia = await db.mediaFile.findMany({
+      where: {
+        uploadStatus: { in: ["PENDING", "FAILED"] },
+        createdAt: { lt: oneHourAgo },
+      },
+      take: 100,
+      select: { id: true },
+    });
+
+    if (staleMedia.length > 0) {
+      const result = await db.mediaFile.deleteMany({
+        where: { id: { in: staleMedia.map((m) => m.id) } },
+      });
+      console.log(JSON.stringify({ level: "info", msg: "Stale media cleaned", deleted: result.count }));
+    }
+  } catch (err) {
+    console.error(JSON.stringify({ level: "error", msg: "Stale media cleanup failed", error: String(err) }));
+  }
+
+  // 2. Soft-delete orphaned media past 24h grace period
+  try {
+    const gracePeriodCutoff = new Date(Date.now() - 24 * 3600000);
+    const result = await db.mediaFile.updateMany({
+      where: {
+        attachedToPost: false,
+        orphanedAt: { lte: gracePeriodCutoff },
+        deletedAt: null,
+      },
+      data: { deletedAt: new Date() },
+    });
+    if (result.count > 0) {
+      console.log(JSON.stringify({ level: "info", msg: "Orphaned media soft-deleted", count: result.count }));
+    }
+  } catch (err) {
+    console.error(JSON.stringify({ level: "error", msg: "Orphaned media cleanup failed", error: String(err) }));
+  }
+
+  // 3. Clean up expired security events
+  try {
+    const result = await db.securityEvent.deleteMany({
+      where: {
+        retentionUntil: { lte: new Date() },
+      },
+    });
+    if (result.count > 0) {
+      console.log(JSON.stringify({ level: "info", msg: "Expired security events cleaned", deleted: result.count }));
+    }
+  } catch (err) {
+    console.error(JSON.stringify({ level: "error", msg: "Security event cleanup failed", error: String(err) }));
+  }
+
+  console.log(JSON.stringify({ level: "info", msg: "Hourly cron complete" }));
+};

package/src/lambda/link-check-worker.ts

@@ -0,0 +1,8 @@
+import type { SQSHandler } from "aws-lambda";
+
+export const handler: SQSHandler = async (event) => {
+  for (const record of event.Records) {
+    console.log(JSON.stringify({ level: "info", msg: "Processing record", messageId: record.messageId }));
+    // TODO: implement
+  }
+};

package/src/lambda/maintenance-cron.ts

@@ -0,0 +1,95 @@
+import { DynamoDBClient, PutItemCommand } from "@aws-sdk/client-dynamodb";
+import { SecretsManagerClient, GetSecretValueCommand } from "@aws-sdk/client-secrets-manager";
+import { marshall } from "@aws-sdk/util-dynamodb";
+import { PrismaClient } from "@prisma/client";
+
+const dynamo = new DynamoDBClient({ region: process.env.AWS_REGION });
+const secretsClient = new SecretsManagerClient({ region: process.env.AWS_REGION });
+const TABLE = process.env.DYNAMODB_TABLE!;
+
+let prisma: PrismaClient | null = null;
+
+async function getPrisma(): Promise<PrismaClient> {
+  if (prisma) return prisma;
+  const secret = await secretsClient.send(new GetSecretValueCommand({ SecretId: process.env.DB_SECRET_ARN! }));
+  const { username, password, host, port, dbname } = JSON.parse(secret.SecretString!);
+  prisma = new PrismaClient({
+    datasources: { db: { url: `postgresql://${username}:${encodeURIComponent(password)}@${host}:${port}/${dbname}?connection_limit=1` } },
+  });
+  return prisma;
+}
+
+export const handler = async (): Promise<void> => {
+  const now = Math.floor(Date.now() / 1000);
+
+  // Acquire cron lock
+  try {
+    await dynamo.send(new PutItemCommand({
+      TableName: TABLE,
+      Item: marshall({
+        pk: "cron:maintenance",
+        sk: "lock",
+        ttl: now + 3600,
+        lockedAt: now,
+      }),
+      ConditionExpression: "attribute_not_exists(pk) OR #ttl < :now",
+      ExpressionAttributeNames: { "#ttl": "ttl" },
+      ExpressionAttributeValues: marshall({ ":now": now }),
+    }));
+  } catch {
+    console.log(JSON.stringify({ level: "info", msg: "Maintenance cron already running, skipping" }));
+    return;
+  }
+
+  console.log(JSON.stringify({ level: "info", msg: "Maintenance cron started" }));
+
+  const db = await getPrisma();
+
+  // 1. Follow counts removed — relationships now live in graph DB (AuraDB)
+  // TODO: Add graph-side consistency check when reconciliation service is wired up
+
+  // 2. Clean up stale DynamoDB cron locks (safety net)
+  try {
+    const lockKeys = ["cron:hourly", "cron:nightly", "cron:maintenance", "cron:cleanup"];
+    for (const pk of lockKeys) {
+      const { GetItemCommand } = await import("@aws-sdk/client-dynamodb");
+      const { unmarshall } = await import("@aws-sdk/util-dynamodb");
+      const result = await dynamo.send(new GetItemCommand({
+        TableName: TABLE,
+        Key: marshall({ pk, sk: "lock" }),
+      }));
+      if (result.Item) {
+        const item = unmarshall(result.Item);
+        // If lock is older than 2 hours, it's stale — delete it
+        if (item.lockedAt && item.lockedAt < now - 7200) {
+          const { DeleteItemCommand } = await import("@aws-sdk/client-dynamodb");
+          await dynamo.send(new DeleteItemCommand({
+            TableName: TABLE,
+            Key: marshall({ pk, sk: "lock" }),
+          }));
+          console.log(JSON.stringify({ level: "info", msg: "Stale cron lock removed", lock: pk }));
+        }
+      }
+    }
+  } catch (err) {
+    console.error(JSON.stringify({ level: "error", msg: "Stale lock cleanup failed", error: String(err) }));
+  }
+
+  // 3. Vacuum analyze critical tables (via advisory lock to prevent concurrent runs)
+  try {
+    await db.$executeRaw`SELECT pg_advisory_lock(42)`;
+    try {
+      await db.$executeRawUnsafe("ANALYZE users");
+      await db.$executeRawUnsafe("ANALYZE posts");
+      await db.$executeRawUnsafe("ANALYZE media_files");
+      await db.$executeRawUnsafe("ANALYZE follows");
+      console.log(JSON.stringify({ level: "info", msg: "ANALYZE completed on critical tables" }));
+    } finally {
+      await db.$executeRaw`SELECT pg_advisory_unlock(42)`;
+    }
+  } catch (err) {
+    console.error(JSON.stringify({ level: "error", msg: "ANALYZE failed", error: String(err) }));
+  }
+
+  console.log(JSON.stringify({ level: "info", msg: "Maintenance cron complete" }));
+};

package/src/lambda/media-processing-worker.ts

@@ -0,0 +1,68 @@
+import type { SQSHandler } from "aws-lambda";
+import { S3Client, GetObjectCommand, PutObjectCommand } from "@aws-sdk/client-s3";
+
+const s3 = new S3Client({ region: process.env.AWS_REGION });
+
+export const handler: SQSHandler = async (event) => {
+  const failedIds: string[] = [];
+
+  for (const record of event.Records) {
+    try {
+      // S3 event notification comes via SQS
+      const s3Event = JSON.parse(record.body);
+      const s3Records = s3Event.Records || [];
+
+      for (const s3Record of s3Records) {
+        const bucket = s3Record.s3.bucket.name;
+        const key = decodeURIComponent(s3Record.s3.object.key.replace(/\+/g, " "));
+
+        if (!key.startsWith("originals/")) continue;
+
+        // Get original
+        const original = await s3.send(new GetObjectCommand({ Bucket: bucket, Key: key }));
+        const chunks: Uint8Array[] = [];
+        for await (const chunk of original.Body as AsyncIterable<Uint8Array>) {
+          chunks.push(chunk);
+        }
+        const buffer = Buffer.concat(chunks);
+
+        // Process with Sharp (must be installed as ARM64 binary)
+        // dynamic import to avoid bundling issues
+        const sharp = (await import("sharp")).default;
+        const hash = key.replace("originals/", "").replace(/\.[^.]+$/, "");
+
+        // Thumbnail: 300px WebP
+        const thumbnail = await sharp(buffer)
+          .resize(300, 300, { fit: "cover" })
+          .webp({ quality: 80 })
+          .toBuffer();
+
+        // Optimized: 1200px WebP
+        const optimized = await sharp(buffer)
+          .resize(1200, 1200, { fit: "inside", withoutEnlargement: true })
+          .webp({ quality: 85 })
+          .toBuffer();
+
+        await Promise.all([
+          s3.send(new PutObjectCommand({
+            Bucket: bucket, Key: `thumbnails/${hash}.webp`,
+            Body: thumbnail, ContentType: "image/webp",
+          })),
+          s3.send(new PutObjectCommand({
+            Bucket: bucket, Key: `optimized/${hash}.webp`,
+            Body: optimized, ContentType: "image/webp",
+          })),
+        ]);
+
+        console.log(JSON.stringify({ level: "info", msg: "Media processed", key, hash }));
+      }
+    } catch (err) {
+      console.error(JSON.stringify({ level: "error", msg: "Media processing failed", err, messageId: record.messageId }));
+      failedIds.push(record.messageId);
+    }
+  }
+
+  if (failedIds.length > 0) {
+    return { batchItemFailures: failedIds.map((id) => ({ itemIdentifier: id })) };
+  }
+};

package/src/lambda/media-reconciliation-worker.ts

@@ -0,0 +1,8 @@
+import type { SQSHandler } from "aws-lambda";
+
+export const handler: SQSHandler = async (event) => {
+  for (const record of event.Records) {
+    console.log(JSON.stringify({ level: "info", msg: "Processing record", messageId: record.messageId }));
+    // TODO: implement
+  }
+};