@projectdochelp/s3te 3.3.3 → 3.4.1

package/README.md

@@ -447,18 +447,22 @@ Once the project is installed, your everyday loop splits into two paths: deploy
 
 1. Edit files in `app/part/` and `app/website/`.
 2. If you use content-driven tags without Webiny, edit `offline/content/en.json` or `offline/content/items.json`.
-3. Validate and render locally.
-4. Run your tests.
-5. Use `deploy` for the first installation or after infrastructure/config/runtime changes.
-6. Use `sync` for day-to-day source publishing into the code buckets.
+3. If you use Webiny and want the current mirrored live content locally, download the content snapshot first.
+4. Validate and render locally.
+5. Run your tests.
+6. Use `deploy` for the first installation or after infrastructure/config/runtime changes.
+7. Use `sync` for day-to-day source publishing into the code buckets.
 
 ```bash
+npx s3te download-content --env dev
 npx s3te validate
 npx s3te render --env dev
 npx s3te test
 npx s3te sync --env dev
 ```
 
+If you are not using Webiny yet, skip `download-content` and keep editing the local JSON files directly.
+
 Use a full deploy only when needed:
 
 ```bash
@@ -480,6 +484,7 @@ Once Webiny is installed and the stack is deployed with Webiny enabled, CMS cont
 | `s3te validate` | Checks config and template syntax without rendering outputs. |
 | `s3te render --env <name>` | Renders locally into `offline/S3TELocal/preview/<env>/...`. |
 | `s3te test` | Runs the project tests from `offline/tests/`. |
+| `s3te download-content --env <name>` | Downloads the mirrored S3TE content table into `offline/content/items.json` for local render and test runs. |
 | `s3te package --env <name>` | Builds the AWS deployment artifacts without deploying them yet. |
 | `s3te sync --env <name>` | Uploads current project sources into the configured code buckets. |
 | `s3te doctor --env <name>` | Checks local machine and AWS access before deploy. |
@@ -1039,6 +1044,16 @@ npx s3te deploy --env prod
 
 That deploy updates the existing environment stack and, when Webiny is enabled, also deploys the separate Webiny option stack for `content-mirror` and its DynamoDB stream mapping. You do not need a fresh S3TE installation. After that, Webiny content changes flow through the deployed AWS resources automatically; only template or asset changes still need `s3te sync --env <name>`.
 
+10. To test current live CMS content locally, pull the mirrored S3TE content table into the local fixture folder before rendering or running tests:
+
+```bash
+npx s3te download-content --env prod
+npx s3te render --env prod
+npx s3te test
+```
+
+`download-content` reads the mirrored S3TE content table, keeps only the newest mirrored item per `contentId` and locale, and writes the result to `offline/content/items.json`. The normal local `dbpart`, `dbmulti`, `dbmultifile`, `dbitem`, and `dbmultifileitem` flow then reads from that file.
+
 Manual versus automatic responsibilities in this step:
 
 - Manual: enable DynamoDB Streams on the Webiny source table

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@projectdochelp/s3te",
-  "version": "3.3.3",
+  "version": "3.4.1",
   "description": "CLI, render core, AWS adapter, and testkit for S3TemplateEngine projects",
   "repository": {
     "type": "git",

package/packages/aws-adapter/src/runtime/content-mirror.mjs

@@ -106,7 +106,27 @@ function serializeStructuredValue(value) {
   return null;
 }
 
+function maybeParseStructuredString(value) {
+  if (typeof value !== "string") {
+    return value;
+  }
+
+  const trimmed = value.trim();
+  if (!trimmed.startsWith("{") && !trimmed.startsWith("[")) {
+    return value;
+  }
+
+  try {
+    return JSON.parse(trimmed);
+  } catch {
+    return value;
+  }
+}
+
 function toSimpleValue(value) {
+  const normalizedValue = maybeParseStructuredString(value);
+
+  value = normalizedValue;
   if (value == null) {
     return null;
   }

package/packages/cli/bin/s3te.mjs

@@ -5,6 +5,7 @@ import path from "node:path";
 import {
   configureProjectOption,
   deployProject,
+  downloadProjectContent,
   doctorProject,
   loadResolvedConfig,
   packageProject,
@@ -82,6 +83,7 @@ function printHelp() {
     "  validate\n" +
     "  render\n" +
     "  test\n" +
+    "  download-content\n" +
     "  package\n" +
     "  sync\n" +
     "  deploy\n" +
@@ -252,6 +254,41 @@ async function main() {
     return;
   }
 
+  if (command === "download-content") {
+    const loaded = await loadConfigForCommand(cwd, options.config);
+    if (!loaded.ok) {
+      if (wantsJson) {
+        printJson("download-content", false, loaded.warnings, loaded.errors, startedAt);
+      } else {
+        for (const error of loaded.errors) {
+          process.stderr.write(`${error.code}: ${error.message}\n`);
+        }
+      }
+      process.exitCode = 2;
+      return;
+    }
+
+    if (!options.env) {
+      process.stderr.write("download-content requires --env <name>\n");
+      process.exitCode = 1;
+      return;
+    }
+
+    const report = await downloadProjectContent(cwd, loaded.config, {
+      environment: asArray(options.env)[0],
+      out: options.out,
+      profile: options.profile
+    });
+
+    if (wantsJson) {
+      printJson("download-content", true, [], [], startedAt, report);
+      return;
+    }
+
+    process.stdout.write(`Downloaded ${report.writtenItems} content item(s) into ${report.outputPath}\n`);
+    return;
+  }
+
   if (command === "package") {
     const loaded = await loadConfigForCommand(cwd, options.config);
     if (!loaded.ok) {

package/packages/cli/src/fs-adapters.mjs

@@ -11,6 +11,39 @@ function normalizeLocale(value) {
   return String(value).trim().toLowerCase();
 }
 
+function comparableTimestamp(value) {
+  if (typeof value === "number" && Number.isFinite(value)) {
+    return value;
+  }
+
+  const timestamp = Date.parse(String(value ?? ""));
+  return Number.isFinite(timestamp) ? timestamp : -1;
+}
+
+function compareContentFreshness(left, right) {
+  const updatedDiff = comparableTimestamp(right.updatedAt) - comparableTimestamp(left.updatedAt);
+  if (updatedDiff !== 0) {
+    return updatedDiff;
+  }
+
+  const changedDiff = comparableTimestamp(right.lastChangedAt) - comparableTimestamp(left.lastChangedAt);
+  if (changedDiff !== 0) {
+    return changedDiff;
+  }
+
+  const createdDiff = comparableTimestamp(right.createdAt) - comparableTimestamp(left.createdAt);
+  if (createdDiff !== 0) {
+    return createdDiff;
+  }
+
+  const versionDiff = Number(right.version ?? -1) - Number(left.version ?? -1);
+  if (versionDiff !== 0) {
+    return versionDiff;
+  }
+
+  return String(right.id ?? "").localeCompare(String(left.id ?? ""));
+}
+
 function buildLocaleCandidates(language, languageLocaleMap) {
   const requested = String(language ?? "").trim();
   const configured = String(languageLocaleMap?.[requested] ?? requested).trim();
@@ -20,7 +53,6 @@ function buildLocaleCandidates(language, languageLocaleMap) {
 function matchesRequestedLocale(item, language, languageLocaleMap) {
   return localeMatchScore(item?.locale, language, languageLocaleMap) > 0;
 }
-
 function localeMatchScore(itemLocale, language, languageLocaleMap) {
   if (!itemLocale) {
     return 1;
@@ -59,7 +91,11 @@ function filterItemsByRequestedLocale(items, language, languageLocaleMap) {
     }
 
     const bestScore = Math.max(...scored.map((entry) => entry.score));
-    return scored.filter((entry) => entry.score === bestScore).map((entry) => entry.item);
+    return scored
+      .filter((entry) => entry.score === bestScore)
+      .map((entry) => entry.item)
+      .sort(compareContentFreshness)
+      .slice(0, 1);
   });
 }
 

package/packages/cli/src/project.mjs

@@ -1,6 +1,8 @@
 import fs from "node:fs/promises";
 import path from "node:path";
 import { spawn } from "node:child_process";
+import { DynamoDBClient } from "@aws-sdk/client-dynamodb";
+import { DynamoDBDocumentClient, ScanCommand } from "@aws-sdk/lib-dynamodb";
 
 import {
   S3teError,
@@ -597,6 +599,114 @@ function normalizeStringList(values) {
     .filter(Boolean))];
 }
 
+function normalizeLocale(value) {
+  return value == null ? "" : String(value).trim().toLowerCase();
+}
+
+function comparableTimestamp(value) {
+  if (typeof value === "number" && Number.isFinite(value)) {
+    return value;
+  }
+
+  const timestamp = Date.parse(String(value ?? ""));
+  return Number.isFinite(timestamp) ? timestamp : -1;
+}
+
+function compareContentFreshness(left, right) {
+  const updatedDiff = comparableTimestamp(right.updatedAt) - comparableTimestamp(left.updatedAt);
+  if (updatedDiff !== 0) {
+    return updatedDiff;
+  }
+
+  const changedDiff = comparableTimestamp(right.lastChangedAt) - comparableTimestamp(left.lastChangedAt);
+  if (changedDiff !== 0) {
+    return changedDiff;
+  }
+
+  const createdDiff = comparableTimestamp(right.createdAt) - comparableTimestamp(left.createdAt);
+  if (createdDiff !== 0) {
+    return createdDiff;
+  }
+
+  const versionDiff = Number(right.version ?? -1) - Number(left.version ?? -1);
+  if (versionDiff !== 0) {
+    return versionDiff;
+  }
+
+  return String(right.id ?? "").localeCompare(String(left.id ?? ""));
+}
+
+function buildContentIdentityKey(item) {
+  return [
+    item.contentId ?? item.id ?? "",
+    item.model ?? "",
+    item.tenant ?? "",
+    normalizeLocale(item.locale)
+  ].join("#");
+}
+
+function compareDownloadedContentOrder(left, right) {
+  return String(left.contentId ?? left.id ?? "").localeCompare(String(right.contentId ?? right.id ?? ""))
+    || normalizeLocale(left.locale).localeCompare(normalizeLocale(right.locale))
+    || String(left.model ?? "").localeCompare(String(right.model ?? ""))
+    || String(left.tenant ?? "").localeCompare(String(right.tenant ?? ""))
+    || compareContentFreshness(left, right);
+}
+
+function deduplicateContentItems(items) {
+  const latestItems = new Map();
+
+  for (const item of items ?? []) {
+    const identityKey = buildContentIdentityKey(item);
+    const current = latestItems.get(identityKey);
+    if (!current || compareContentFreshness(item, current) < 0) {
+      latestItems.set(identityKey, item);
+    }
+  }
+
+  return [...latestItems.values()].sort(compareDownloadedContentOrder);
+}
+
+async function scanRemoteContentTable({ tableName, region, profile }) {
+  const previousProfile = process.env.AWS_PROFILE;
+  if (profile) {
+    process.env.AWS_PROFILE = profile;
+  }
+
+  const baseClient = new DynamoDBClient({ region });
+  const documentClient = DynamoDBDocumentClient.from(baseClient);
+  const items = [];
+  let lastEvaluatedKey;
+
+  try {
+    do {
+      const response = await documentClient.send(new ScanCommand({
+        TableName: tableName,
+        ExclusiveStartKey: lastEvaluatedKey
+      }));
+      items.push(...(response.Items ?? []));
+      lastEvaluatedKey = response.LastEvaluatedKey;
+    } while (lastEvaluatedKey);
+
+    return items;
+  } catch (error) {
+    throw new S3teError("AWS_AUTH_ERROR", `Unable to download content from DynamoDB table ${tableName}.`, {
+      tableName,
+      region,
+      cause: error.message
+    });
+  } finally {
+    baseClient.destroy();
+    if (profile) {
+      if (previousProfile === undefined) {
+        delete process.env.AWS_PROFILE;
+      } else {
+        process.env.AWS_PROFILE = previousProfile;
+      }
+    }
+  }
+}
+
 export async function loadResolvedConfig(projectDir, configPath) {
   const rawConfig = await loadProjectConfig(configPath);
   const result = await validateAndResolveProjectConfig(rawConfig, { projectDir });
@@ -861,6 +971,34 @@ export async function runProjectTests(projectDir) {
   });
 }
 
+export async function downloadProjectContent(projectDir, config, options = {}) {
+  assertKnownEnvironment(config, options.environment);
+  const runtimeConfig = buildEnvironmentRuntimeConfig(config, options.environment);
+  const tableName = runtimeConfig.tables.content;
+  const region = runtimeConfig.awsRegion;
+  const outputPath = path.resolve(projectDir, options.out ?? path.join("offline", "content", "items.json"));
+  const scanContentItemsFn = options.scanContentItemsFn ?? scanRemoteContentTable;
+
+  const remoteItems = await scanContentItemsFn({
+    tableName,
+    region,
+    profile: options.profile
+  });
+  const items = deduplicateContentItems(remoteItems);
+
+  await writeTextFile(outputPath, JSON.stringify(items, null, 2) + "\n");
+
+  return {
+    environment: options.environment,
+    region,
+    tableName,
+    outputPath: normalizePath(path.relative(projectDir, outputPath)),
+    downloadedItems: remoteItems.length,
+    writtenItems: items.length,
+    deduplicatedItems: Math.max(0, remoteItems.length - items.length)
+  };
+}
+
 export async function packageProject(projectDir, config, options = {}) {
   assertKnownEnvironment(config, options.environment);
   return packageAwsProject({

package/packages/core/src/content-query.mjs

@@ -22,6 +22,23 @@ function legacyAttributeValueToPlain(attribute) {
   return undefined;
 }
 
+function maybeParseStructuredString(value) {
+  if (typeof value !== "string") {
+    return value;
+  }
+
+  const trimmed = value.trim();
+  if (!trimmed.startsWith("{") && !trimmed.startsWith("[")) {
+    return value;
+  }
+
+  try {
+    return JSON.parse(trimmed);
+  } catch {
+    return value;
+  }
+}
+
 function readComparableValue(item, field) {
   if (field === "__typename") {
     return item.model;
@@ -159,12 +176,14 @@ export function readContentField(item, field, language) {
 }
 
 export function serializeContentValue(value) {
-  if (value == null) {
+  const normalizedValue = maybeParseStructuredString(value);
+
+  if (normalizedValue == null) {
     return "";
   }
 
-  if (Array.isArray(value)) {
-    return value.map((entry) => {
+  if (Array.isArray(normalizedValue)) {
+    return normalizedValue.map((entry) => {
       const text = typeof entry === "string" && entry.includes("-")
         ? entry.slice(entry.lastIndexOf("-") + 1)
         : String(entry);
@@ -172,5 +191,14 @@ export function serializeContentValue(value) {
     }).join("");
   }
 
-  return String(value);
+  if (normalizedValue && typeof normalizedValue === "object") {
+    if (typeof normalizedValue.html === "string") {
+      return normalizedValue.html;
+    }
+    if (typeof normalizedValue.text === "string") {
+      return normalizedValue.text;
+    }
+  }
+
+  return String(normalizedValue);
 }