@projectdochelp/s3te 3.3.2 → 3.4.0

package/README.md

@@ -447,18 +447,22 @@ Once the project is installed, your everyday loop splits into two paths: deploy
 
 1. Edit files in `app/part/` and `app/website/`.
 2. If you use content-driven tags without Webiny, edit `offline/content/en.json` or `offline/content/items.json`.
-3. Validate and render locally.
-4. Run your tests.
-5. Use `deploy` for the first installation or after infrastructure/config/runtime changes.
-6. Use `sync` for day-to-day source publishing into the code buckets.
+3. If you use Webiny and want the current mirrored live content locally, download the content snapshot first.
+4. Validate and render locally.
+5. Run your tests.
+6. Use `deploy` for the first installation or after infrastructure/config/runtime changes.
+7. Use `sync` for day-to-day source publishing into the code buckets.
 
 ```bash
+npx s3te download-content --env dev
 npx s3te validate
 npx s3te render --env dev
 npx s3te test
 npx s3te sync --env dev
 ```
 
+If you are not using Webiny yet, skip `download-content` and keep editing the local JSON files directly.
+
 Use a full deploy only when needed:
 
 ```bash
@@ -480,6 +484,7 @@ Once Webiny is installed and the stack is deployed with Webiny enabled, CMS cont
 | `s3te validate` | Checks config and template syntax without rendering outputs. |
 | `s3te render --env <name>` | Renders locally into `offline/S3TELocal/preview/<env>/...`. |
 | `s3te test` | Runs the project tests from `offline/tests/`. |
+| `s3te download-content --env <name>` | Downloads the mirrored S3TE content table into `offline/content/items.json` for local render and test runs. |
 | `s3te package --env <name>` | Builds the AWS deployment artifacts without deploying them yet. |
 | `s3te sync --env <name>` | Uploads current project sources into the configured code buckets. |
 | `s3te doctor --env <name>` | Checks local machine and AWS access before deploy. |
@@ -1039,6 +1044,16 @@ npx s3te deploy --env prod
 
 That deploy updates the existing environment stack and, when Webiny is enabled, also deploys the separate Webiny option stack for `content-mirror` and its DynamoDB stream mapping. You do not need a fresh S3TE installation. After that, Webiny content changes flow through the deployed AWS resources automatically; only template or asset changes still need `s3te sync --env <name>`.
 
+10. To test current live CMS content locally, pull the mirrored S3TE content table into the local fixture folder before rendering or running tests:
+
+```bash
+npx s3te download-content --env prod
+npx s3te render --env prod
+npx s3te test
+```
+
+`download-content` reads the mirrored S3TE content table, keeps only the newest mirrored item per `contentId` and locale, and writes the result to `offline/content/items.json`. The normal local `dbpart`, `dbmulti`, `dbmultifile`, `dbitem`, and `dbmultifileitem` flow then reads from that file.
+
 Manual versus automatic responsibilities in this step:
 
 - Manual: enable DynamoDB Streams on the Webiny source table

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@projectdochelp/s3te",
-  "version": "3.3.2",
+  "version": "3.4.0",
   "description": "CLI, render core, AWS adapter, and testkit for S3TemplateEngine projects",
   "repository": {
     "type": "git",

package/packages/aws-adapter/src/runtime/common.mjs

@@ -76,8 +76,37 @@ function localeMatchScore(itemLocale, language, languageLocaleMap) {
   return 0;
 }
 
-function matchesRequestedLocale(item, language, languageLocaleMap) {
-  return localeMatchScore(item?.locale, language, languageLocaleMap) > 0;
+function comparableTimestamp(value) {
+  if (typeof value === "number" && Number.isFinite(value)) {
+    return value;
+  }
+
+  const timestamp = Date.parse(String(value ?? ""));
+  return Number.isFinite(timestamp) ? timestamp : -1;
+}
+
+function compareContentFreshness(left, right) {
+  const updatedDiff = comparableTimestamp(right.updatedAt) - comparableTimestamp(left.updatedAt);
+  if (updatedDiff !== 0) {
+    return updatedDiff;
+  }
+
+  const changedDiff = comparableTimestamp(right.lastChangedAt) - comparableTimestamp(left.lastChangedAt);
+  if (changedDiff !== 0) {
+    return changedDiff;
+  }
+
+  const createdDiff = comparableTimestamp(right.createdAt) - comparableTimestamp(left.createdAt);
+  if (createdDiff !== 0) {
+    return createdDiff;
+  }
+
+  const versionDiff = Number(right.version ?? -1) - Number(left.version ?? -1);
+  if (versionDiff !== 0) {
+    return versionDiff;
+  }
+
+  return String(right.id ?? "").localeCompare(String(left.id ?? ""));
 }
 
 function filterItemsByRequestedLocale(items, language, languageLocaleMap) {
@@ -103,7 +132,11 @@ function filterItemsByRequestedLocale(items, language, languageLocaleMap) {
     }
 
     const bestScore = Math.max(...scored.map((entry) => entry.score));
-    return scored.filter((entry) => entry.score === bestScore).map((entry) => entry.item);
+    return scored
+      .filter((entry) => entry.score === bestScore)
+      .map((entry) => entry.item)
+      .sort(compareContentFreshness)
+      .slice(0, 1);
   });
 }
 
@@ -390,14 +423,7 @@ export class DynamoContentRepository {
       }
     }).promise();
     const items = response.Items ?? [];
-    const candidates = items
-      .map((item) => ({
-        item,
-        score: localeMatchScore(item.locale, language, this.languageLocaleMap)
-      }))
-      .filter((entry) => entry.score > 0)
-      .sort((left, right) => right.score - left.score || String(left.item.id).localeCompare(String(right.item.id)));
-    return candidates[0]?.item ?? null;
+    return filterItemsByRequestedLocale(items, language, this.languageLocaleMap)[0] ?? null;
   }
 
   async query(query, language) {

package/packages/aws-adapter/src/runtime/content-mirror.mjs

@@ -314,6 +314,17 @@ function extractWebinyTenant(item) {
     ?? null;
 }
 
+function normalizeMirrorLocale(value) {
+  return value == null ? "" : String(value).trim().toLowerCase();
+}
+
+function isSameMirroredContentIdentity(existingItem, contentItem) {
+  return String(existingItem.contentId ?? "") === String(contentItem.contentId ?? "")
+    && String(existingItem.model ?? "") === String(contentItem.model ?? "")
+    && String(existingItem.tenant ?? "") === String(contentItem.tenant ?? "")
+    && normalizeMirrorLocale(existingItem.locale) === normalizeMirrorLocale(contentItem.locale);
+}
+
 export function normalizeContentItem(item, options = {}) {
   const root = getItemRoot(item);
   const values = normalizeValues(item, options.modelFields);
@@ -369,12 +380,53 @@ async function loadModelFields(clients, sourceTableName, tenant, modelId, cache)
   return fields;
 }
 
+async function listMirroredContentItems(clients, tableName, indexName, contentId) {
+  const items = [];
+  let lastEvaluatedKey;
+
+  do {
+    const response = await clients.dynamo.query({
+      TableName: tableName,
+      IndexName: indexName,
+      KeyConditionExpression: "contentId = :contentId",
+      ExpressionAttributeValues: {
+        ":contentId": contentId
+      },
+      ExclusiveStartKey: lastEvaluatedKey
+    }).promise();
+    items.push(...(response.Items ?? []));
+    lastEvaluatedKey = response.LastEvaluatedKey;
+  } while (lastEvaluatedKey);
+
+  return items;
+}
+
+async function removeMirroredContentRevisions(clients, tableName, indexName, contentItem, excludeId = null) {
+  const mirroredItems = await listMirroredContentItems(clients, tableName, indexName, contentItem.contentId);
+  const removals = mirroredItems.filter((existingItem) => (
+    isSameMirroredContentIdentity(existingItem, contentItem)
+    && existingItem.id !== excludeId
+  ));
+
+  for (const existingItem of removals) {
+    await clients.dynamo.delete({
+      TableName: tableName,
+      Key: {
+        id: existingItem.id
+      }
+    }).promise();
+  }
+
+  return removals.length;
+}
+
 export async function handler(event) {
   const clients = createAwsClients();
   const tableName = process.env.S3TE_CONTENT_TABLE;
   const renderWorkerName = process.env.S3TE_RENDER_WORKER_NAME;
   const environmentName = process.env.S3TE_ENVIRONMENT;
   const sourceTableName = process.env.S3TE_WEBINY_SOURCE_TABLE;
+  const contentIndexName = process.env.S3TE_CONTENT_ID_INDEX_NAME ?? "contentid";
   const configuredTenant = String(process.env.S3TE_WEBINY_TENANT ?? "").trim();
   const relevantModels = new Set(String(process.env.S3TE_RELEVANT_MODELS ?? "")
     .split(",")
@@ -411,13 +463,7 @@ export async function handler(event) {
 
     const shouldDelete = record.eventName === "REMOVE" || !isPublished(item);
     if (shouldDelete) {
-      await clients.dynamo.delete({
-        TableName: tableName,
-        Key: {
-          id: contentItem.id
-        }
-      }).promise();
-      deleted += 1;
+      deleted += await removeMirroredContentRevisions(clients, tableName, contentIndexName, contentItem);
       await invokeLambdaEvent(clients.lambda, renderWorkerName, {
         type: "content-item",
         action: "delete",
@@ -429,6 +475,7 @@ export async function handler(event) {
       continue;
     }
 
+    await removeMirroredContentRevisions(clients, tableName, contentIndexName, contentItem, contentItem.id);
     await clients.dynamo.put({
       TableName: tableName,
       Item: contentItem

package/packages/aws-adapter/src/template.mjs

@@ -631,6 +631,7 @@ export function buildWebinyCloudFormationTemplate({ config, environment }) {
             Variables: {
               S3TE_ENVIRONMENT: environment,
               S3TE_CONTENT_TABLE: runtimeConfig.tables.content,
+              S3TE_CONTENT_ID_INDEX_NAME: config.aws.contentStore.contentIdIndexName,
               S3TE_RELEVANT_MODELS: runtimeConfig.integrations.webiny.relevantModels.join(","),
               S3TE_WEBINY_SOURCE_TABLE: runtimeConfig.integrations.webiny.sourceTableName,
               S3TE_WEBINY_TENANT: runtimeConfig.integrations.webiny.tenant ?? "",

package/packages/cli/bin/s3te.mjs

@@ -5,6 +5,7 @@ import path from "node:path";
 import {
   configureProjectOption,
   deployProject,
+  downloadProjectContent,
   doctorProject,
   loadResolvedConfig,
   packageProject,
@@ -82,6 +83,7 @@ function printHelp() {
     "  validate\n" +
     "  render\n" +
     "  test\n" +
+    "  download-content\n" +
     "  package\n" +
     "  sync\n" +
     "  deploy\n" +
@@ -252,6 +254,41 @@ async function main() {
     return;
   }
 
+  if (command === "download-content") {
+    const loaded = await loadConfigForCommand(cwd, options.config);
+    if (!loaded.ok) {
+      if (wantsJson) {
+        printJson("download-content", false, loaded.warnings, loaded.errors, startedAt);
+      } else {
+        for (const error of loaded.errors) {
+          process.stderr.write(`${error.code}: ${error.message}\n`);
+        }
+      }
+      process.exitCode = 2;
+      return;
+    }
+
+    if (!options.env) {
+      process.stderr.write("download-content requires --env <name>\n");
+      process.exitCode = 1;
+      return;
+    }
+
+    const report = await downloadProjectContent(cwd, loaded.config, {
+      environment: asArray(options.env)[0],
+      out: options.out,
+      profile: options.profile
+    });
+
+    if (wantsJson) {
+      printJson("download-content", true, [], [], startedAt, report);
+      return;
+    }
+
+    process.stdout.write(`Downloaded ${report.writtenItems} content item(s) into ${report.outputPath}\n`);
+    return;
+  }
+
   if (command === "package") {
     const loaded = await loadConfigForCommand(cwd, options.config);
     if (!loaded.ok) {

package/packages/cli/src/fs-adapters.mjs

@@ -11,6 +11,39 @@ function normalizeLocale(value) {
   return String(value).trim().toLowerCase();
 }
 
+function comparableTimestamp(value) {
+  if (typeof value === "number" && Number.isFinite(value)) {
+    return value;
+  }
+
+  const timestamp = Date.parse(String(value ?? ""));
+  return Number.isFinite(timestamp) ? timestamp : -1;
+}
+
+function compareContentFreshness(left, right) {
+  const updatedDiff = comparableTimestamp(right.updatedAt) - comparableTimestamp(left.updatedAt);
+  if (updatedDiff !== 0) {
+    return updatedDiff;
+  }
+
+  const changedDiff = comparableTimestamp(right.lastChangedAt) - comparableTimestamp(left.lastChangedAt);
+  if (changedDiff !== 0) {
+    return changedDiff;
+  }
+
+  const createdDiff = comparableTimestamp(right.createdAt) - comparableTimestamp(left.createdAt);
+  if (createdDiff !== 0) {
+    return createdDiff;
+  }
+
+  const versionDiff = Number(right.version ?? -1) - Number(left.version ?? -1);
+  if (versionDiff !== 0) {
+    return versionDiff;
+  }
+
+  return String(right.id ?? "").localeCompare(String(left.id ?? ""));
+}
+
 function buildLocaleCandidates(language, languageLocaleMap) {
   const requested = String(language ?? "").trim();
   const configured = String(languageLocaleMap?.[requested] ?? requested).trim();
@@ -20,7 +53,6 @@ function buildLocaleCandidates(language, languageLocaleMap) {
 function matchesRequestedLocale(item, language, languageLocaleMap) {
   return localeMatchScore(item?.locale, language, languageLocaleMap) > 0;
 }
-
 function localeMatchScore(itemLocale, language, languageLocaleMap) {
   if (!itemLocale) {
     return 1;
@@ -59,7 +91,11 @@ function filterItemsByRequestedLocale(items, language, languageLocaleMap) {
     }
 
     const bestScore = Math.max(...scored.map((entry) => entry.score));
-    return scored.filter((entry) => entry.score === bestScore).map((entry) => entry.item);
+    return scored
+      .filter((entry) => entry.score === bestScore)
+      .map((entry) => entry.item)
+      .sort(compareContentFreshness)
+      .slice(0, 1);
   });
 }
 

package/packages/cli/src/project.mjs

@@ -1,6 +1,8 @@
 import fs from "node:fs/promises";
 import path from "node:path";
 import { spawn } from "node:child_process";
+import { DynamoDBClient } from "@aws-sdk/client-dynamodb";
+import { DynamoDBDocumentClient, ScanCommand } from "@aws-sdk/lib-dynamodb";
 
 import {
   S3teError,
@@ -597,6 +599,114 @@ function normalizeStringList(values) {
     .filter(Boolean))];
 }
 
+function normalizeLocale(value) {
+  return value == null ? "" : String(value).trim().toLowerCase();
+}
+
+function comparableTimestamp(value) {
+  if (typeof value === "number" && Number.isFinite(value)) {
+    return value;
+  }
+
+  const timestamp = Date.parse(String(value ?? ""));
+  return Number.isFinite(timestamp) ? timestamp : -1;
+}
+
+function compareContentFreshness(left, right) {
+  const updatedDiff = comparableTimestamp(right.updatedAt) - comparableTimestamp(left.updatedAt);
+  if (updatedDiff !== 0) {
+    return updatedDiff;
+  }
+
+  const changedDiff = comparableTimestamp(right.lastChangedAt) - comparableTimestamp(left.lastChangedAt);
+  if (changedDiff !== 0) {
+    return changedDiff;
+  }
+
+  const createdDiff = comparableTimestamp(right.createdAt) - comparableTimestamp(left.createdAt);
+  if (createdDiff !== 0) {
+    return createdDiff;
+  }
+
+  const versionDiff = Number(right.version ?? -1) - Number(left.version ?? -1);
+  if (versionDiff !== 0) {
+    return versionDiff;
+  }
+
+  return String(right.id ?? "").localeCompare(String(left.id ?? ""));
+}
+
+function buildContentIdentityKey(item) {
+  return [
+    item.contentId ?? item.id ?? "",
+    item.model ?? "",
+    item.tenant ?? "",
+    normalizeLocale(item.locale)
+  ].join("#");
+}
+
+function compareDownloadedContentOrder(left, right) {
+  return String(left.contentId ?? left.id ?? "").localeCompare(String(right.contentId ?? right.id ?? ""))
+    || normalizeLocale(left.locale).localeCompare(normalizeLocale(right.locale))
+    || String(left.model ?? "").localeCompare(String(right.model ?? ""))
+    || String(left.tenant ?? "").localeCompare(String(right.tenant ?? ""))
+    || compareContentFreshness(left, right);
+}
+
+function deduplicateContentItems(items) {
+  const latestItems = new Map();
+
+  for (const item of items ?? []) {
+    const identityKey = buildContentIdentityKey(item);
+    const current = latestItems.get(identityKey);
+    if (!current || compareContentFreshness(item, current) < 0) {
+      latestItems.set(identityKey, item);
+    }
+  }
+
+  return [...latestItems.values()].sort(compareDownloadedContentOrder);
+}
+
+async function scanRemoteContentTable({ tableName, region, profile }) {
+  const previousProfile = process.env.AWS_PROFILE;
+  if (profile) {
+    process.env.AWS_PROFILE = profile;
+  }
+
+  const baseClient = new DynamoDBClient({ region });
+  const documentClient = DynamoDBDocumentClient.from(baseClient);
+  const items = [];
+  let lastEvaluatedKey;
+
+  try {
+    do {
+      const response = await documentClient.send(new ScanCommand({
+        TableName: tableName,
+        ExclusiveStartKey: lastEvaluatedKey
+      }));
+      items.push(...(response.Items ?? []));
+      lastEvaluatedKey = response.LastEvaluatedKey;
+    } while (lastEvaluatedKey);
+
+    return items;
+  } catch (error) {
+    throw new S3teError("AWS_AUTH_ERROR", `Unable to download content from DynamoDB table ${tableName}.`, {
+      tableName,
+      region,
+      cause: error.message
+    });
+  } finally {
+    baseClient.destroy();
+    if (profile) {
+      if (previousProfile === undefined) {
+        delete process.env.AWS_PROFILE;
+      } else {
+        process.env.AWS_PROFILE = previousProfile;
+      }
+    }
+  }
+}
+
 export async function loadResolvedConfig(projectDir, configPath) {
   const rawConfig = await loadProjectConfig(configPath);
   const result = await validateAndResolveProjectConfig(rawConfig, { projectDir });
@@ -861,6 +971,34 @@ export async function runProjectTests(projectDir) {
   });
 }
 
+export async function downloadProjectContent(projectDir, config, options = {}) {
+  assertKnownEnvironment(config, options.environment);
+  const runtimeConfig = buildEnvironmentRuntimeConfig(config, options.environment);
+  const tableName = runtimeConfig.tables.content;
+  const region = runtimeConfig.awsRegion;
+  const outputPath = path.resolve(projectDir, options.out ?? path.join("offline", "content", "items.json"));
+  const scanContentItemsFn = options.scanContentItemsFn ?? scanRemoteContentTable;
+
+  const remoteItems = await scanContentItemsFn({
+    tableName,
+    region,
+    profile: options.profile
+  });
+  const items = deduplicateContentItems(remoteItems);
+
+  await writeTextFile(outputPath, JSON.stringify(items, null, 2) + "\n");
+
+  return {
+    environment: options.environment,
+    region,
+    tableName,
+    outputPath: normalizePath(path.relative(projectDir, outputPath)),
+    downloadedItems: remoteItems.length,
+    writtenItems: items.length,
+    deduplicatedItems: Math.max(0, remoteItems.length - items.length)
+  };
+}
+
 export async function packageProject(projectDir, config, options = {}) {
   assertKnownEnvironment(config, options.environment);
   return packageAwsProject({