hubspot-cms-sync 0.1.0

package/src/lib/canonical.mjs

@@ -0,0 +1,234 @@
+// sync/lib/canonical.mjs
+//
+// Canonicalization for clean, portable git diffs in the bidirectional
+// git <-> HubSpot sync system.
+//
+// PURE: no I/O, no network, no `process`/`fs`. Every export is a pure
+// function so it can be unit-tested without a HubSpot account (per the
+// codex review tier-1 requirement). Higher layers (adapters) do the I/O
+// and call into these helpers.
+//
+// ─────────────────────────────────────────────────────────────────────────
+// WHY canonicalization is RESOURCE/FIELD-SPECIFIC, not a blanket omit
+// (codex finding #8, design §1.3 / §6.3):
+//
+// The naive approach — "recursively drop every null/'' and every empty
+// object" — is WRONG for this corpus, and the codex review flagged it as a
+// must-fix. Page MODULE CONTENT ("widget carriers": widgets /
+// widgetContainers / layoutSections) is pushed back to HubSpot with a
+// REPLACE-not-merge PATCH semantics: the complete carrier object is sent and
+// HubSpot overwrites the whole widget, it does NOT merge field-by-field.
+// That means a widget value of `css: {}`, `child_css: {}`, or `label: ""`
+// is LOAD-BEARING — if canonicalization silently dropped it, the round-trip
+// (pull -> canonical -> push) would send a thinner payload than HubSpot
+// expects and blank-out / reset rendered styling. So inside widget carriers
+// we KEEP empty css/child_css/label/body. The volatile-key stripping
+// (stripVolatile) is therefore a *targeted* removal of per-account/volatile
+// keys (ids, timestamps, urls, hashes, ...), NOT a "remove all empties" pass.
+//
+// Each exported transform documents exactly which fields it touches.
+// ─────────────────────────────────────────────────────────────────────────
+
+// Keys that are per-account or volatile and must NEVER land in a committed
+// canonical file. Matched by exact name. *At / *ById are matched by suffix
+// (see SUFFIX_AT / SUFFIX_BY_ID below) so new timestamp variants
+// (publishDate, archivedAt, lastEditedAt, ...) are caught without an
+// ever-growing list.
+const VOLATILE_EXACT = new Set([
+  'id',
+  'currentState',
+  'url',
+  'hash',
+  'folder',
+  'children',
+  // publishDate is volatile per design §1.3 (HubSpot recomputes it on
+  // publish); it ends in "Date" not "At", so it is listed explicitly here.
+  'publishDate',
+]);
+
+// Suffix matchers for the families of volatile keys.
+//  *At   -> createdAt, updatedAt, archivedAt, ...
+//  *ById -> createdById, updatedById, ...
+function isVolatileKey(key, extra) {
+  if (VOLATILE_EXACT.has(key)) return true;
+  if (extra.has(key)) return true;
+  if (key.endsWith('ById')) return true;
+  // `...At` timestamp family (createdAt/updatedAt/archivedAt/...). Guard on
+  // length so a literal key named "At" (none in this corpus) doesn't trip,
+  // and avoid matching "...Cat"/"...Format" by requiring an uppercase boundary.
+  if (/[a-z0-9]At$/.test(key)) return true;
+  return false;
+}
+
+/**
+ * stableStringify(obj) -> string
+ *
+ * Deterministic JSON serialization for diff-clean commits:
+ *   - object keys recursively SORTED (lexicographic, default JS string sort)
+ *   - 2-space indent
+ *   - LF line endings (JSON.stringify never emits CRLF)
+ *   - trailing newline
+ *   - arrays keep their order (order is meaningful: layoutSection rows,
+ *     stats, logos, ...)
+ *
+ * Does NOT strip or normalize values — pair it with stripVolatile /
+ * canonicalPage first. The only transform is key ordering + formatting, so
+ * two semantically-equal but differently-key-ordered inputs serialize byte
+ * identically.
+ */
+export function stableStringify(obj) {
+  return JSON.stringify(sortKeysDeep(obj), null, 2) + '\n';
+}
+
+// Recursively return a structurally-equal value with all object keys sorted.
+// Arrays are mapped in place (order preserved). Primitives pass through.
+function sortKeysDeep(value) {
+  if (Array.isArray(value)) return value.map(sortKeysDeep);
+  if (value && typeof value === 'object') {
+    const out = {};
+    for (const key of Object.keys(value).sort()) {
+      out[key] = sortKeysDeep(value[key]);
+    }
+    return out;
+  }
+  return value;
+}
+
+/**
+ * stripVolatile(obj, extraKeys=[]) -> obj
+ *
+ * Recursively removes per-account / volatile keys so the canonical tree
+ * carries no portal-specific identity. Returns a NEW value (does not mutate
+ * the input). Recurses through both objects and arrays.
+ *
+ * Removed: id, *At (createdAt/updatedAt/archivedAt/publishDate-family via
+ * explicit + suffix match), *ById, currentState, url, hash, folder, children,
+ * plus any names passed in `extraKeys`.
+ *
+ * This is a TARGETED key removal, NOT an empty/null omit (see the file
+ * header). Empty objects/strings that remain after stripping are preserved.
+ */
+export function stripVolatile(obj, extraKeys = []) {
+  const extra = extraKeys instanceof Set ? extraKeys : new Set(extraKeys);
+  return strip(obj, extra);
+}
+
+function strip(value, extra) {
+  if (Array.isArray(value)) return value.map((v) => strip(v, extra));
+  if (value && typeof value === 'object') {
+    const out = {};
+    for (const [key, v] of Object.entries(value)) {
+      if (isVolatileKey(key, extra)) continue;
+      out[key] = strip(v, extra);
+    }
+    return out;
+  }
+  return value;
+}
+
+/**
+ * slugToFile(slug) -> filename
+ *
+ * Maps a HubSpot page slug to a safe canonical filename stem.
+ *   - '' (homepage)        -> 'home'
+ *   - '/' inside the slug  -> '__'   (e.g. 'blog/x' -> 'blog__x')
+ *
+ * The empty-slug homepage cannot be a filename, so it gets the 'home'
+ * sentinel. '/' is not legal in a path segment, so it is escaped to '__'.
+ */
+export function slugToFile(slug) {
+  if (slug === '' || slug == null) return 'home';
+  return String(slug).replace(/\//g, '__');
+}
+
+/**
+ * fileToSlug(name) -> slug
+ *
+ * Inverse of slugToFile.
+ *   - 'home'      -> ''
+ *   - 'blog__x'   -> 'blog/x'
+ *
+ * A trailing '.json' (or any extension) should be stripped by the caller
+ * before calling; fileToSlug operates on the bare stem.
+ */
+export function fileToSlug(name) {
+  if (name === 'home' || name === '' || name == null) return '';
+  return String(name).replace(/__/g, '/');
+}
+
+// Fields kept on a canonical page widget carrier value. Per codex finding #8
+// these are kept VERBATIM even when empty (css/child_css/label), because the
+// push is replace-not-merge.
+const WIDGET_KEEP = ['body', 'name', 'type', 'label', 'css', 'child_css'];
+
+/**
+ * canonicalPage(rawPage) -> canonical page definition
+ *
+ * Produces the portable, slug-keyed page DEFINITION object:
+ *   { slug, name, htmlTitle, metaDescription, language, templatePath, widgets }
+ *
+ * - slug:            raw slug, defaulting to '' (homepage). NOT the volatile id.
+ * - name/htmlTitle/metaDescription: SEO/definition fields, defaulted to ''.
+ * - language:        defaults to 'en'.
+ * - templatePath:    kept as-is (the manifest-driven rewrite of non-portable
+ *                    marketplace/generated paths is a separate adapter
+ *                    concern, intentionally NOT done here so this stays pure).
+ * - widgets:         normalized widget carrier map (see normalizeWidgets).
+ *
+ * Volatile page-level keys (id, url, *At, currentState, publishDate, ...) are
+ * simply not projected here. We project an explicit allow-list of definition
+ * fields rather than strip-everything-else, so the schema is stable and
+ * documented.
+ */
+export function canonicalPage(rawPage) {
+  const raw = rawPage || {};
+  return {
+    slug: raw.slug ?? '',
+    name: raw.name ?? '',
+    htmlTitle: raw.htmlTitle ?? '',
+    metaDescription: raw.metaDescription ?? '',
+    language: raw.language ?? 'en',
+    templatePath: raw.templatePath ?? '',
+    widgets: normalizeWidgets(raw.widgets),
+  };
+}
+
+/**
+ * normalizeWidgets(widgets) -> normalized widget carrier map
+ *
+ * For each widget instance (keyed by instance name), keep exactly the
+ * carrier fields HubSpot needs for a replace-not-merge PATCH:
+ *   body, name, type, label, css, child_css
+ *
+ * CRITICAL (codex #8): empty css/child_css/label and body sub-fields are
+ * KEPT, not omitted. `body` is passed through unchanged (its per-field values
+ * — including empty strings like section_id:'' — are the actual content and
+ * must survive). Only keys OUTSIDE the WIDGET_KEEP set (e.g. a stray volatile
+ * `id` HubSpot might echo on a widget) are dropped.
+ *
+ * Default empties are supplied for any missing carrier field so the pushed
+ * payload is always complete: label -> '', css/child_css -> {}, type ->
+ * 'module', body -> {}.
+ */
+export function normalizeWidgets(widgets) {
+  if (!widgets || typeof widgets !== 'object') return {};
+  const out = {};
+  for (const [instanceName, raw] of Object.entries(widgets)) {
+    if (!raw || typeof raw !== 'object') continue;
+    const w = {};
+    for (const field of WIDGET_KEEP) {
+      if (field in raw) {
+        w[field] = raw[field];
+      } else {
+        // Supply explicit empties so the carrier is complete for push.
+        if (field === 'css' || field === 'child_css') w[field] = {};
+        else if (field === 'label') w[field] = '';
+        else if (field === 'type') w[field] = 'module';
+        else if (field === 'name') w[field] = instanceName;
+        else if (field === 'body') w[field] = {};
+      }
+    }
+    out[instanceName] = w;
+  }
+  return out;
+}

package/src/lib/hub.mjs

@@ -0,0 +1,197 @@
+// sync/lib/hub.mjs — account + HTTP layer for HubSpot bidirectional sync.
+//
+// The account/HTTP plumbing shared by every adapter and orchestrator. Pure,
+// importable, unit-testable: account resolution and paging accumulation do not
+// touch the network (paging is driven through an injectable hub function), and
+// slug matching is a pure helper over a fetched list.
+//
+// Accounts: sync/accounts.json maps name -> { portalId, label }.
+// Keys: per-portal service keys (Bearer) at $HUBSPOT_KEY_DIR/<portalId>.key
+//       (default ~/.hubspot/<portalId>.key). Never committed.
+//
+// Production (portalId 529456) is READ-ONLY. This module provides NO default
+// that writes to prod — callers pass an explicit account object every time.
+
+import { readFileSync, existsSync } from 'node:fs';
+import { join, dirname } from 'node:path';
+import { homedir } from 'node:os';
+import { fileURLToPath } from 'node:url';
+import { loadConfigSyncFallback } from '../config.mjs';
+
+const API = 'https://api.hubapi.com';
+
+const __dirname = dirname(fileURLToPath(import.meta.url));
+
+function fallbackConfig() {
+  return loadConfigSyncFallback();
+}
+
+function keyDir(cfg = fallbackConfig()) {
+  return cfg.keyDir || process.env[cfg.keyDirEnv || 'HUBSPOT_KEY_DIR'] || join(homedir(), '.hubspot');
+}
+
+/**
+ * Load and parse sync/accounts.json.
+ * @returns {object} parsed account registry (name -> { portalId, label })
+ */
+export function loadAccounts(cfg = fallbackConfig()) {
+  const accountsPath = cfg.accountsPath || join(cfg.root || process.cwd(), cfg.accountsFile || 'sync/accounts.json');
+  let text;
+  try {
+    text = readFileSync(accountsPath, 'utf8');
+  } catch (e) {
+    throw new Error(`Cannot read accounts file at ${accountsPath}: ${e.message}`);
+  }
+  try {
+    return JSON.parse(text);
+  } catch (e) {
+    throw new Error(`Invalid JSON in ${accountsPath}: ${e.message}`);
+  }
+}
+
+/**
+ * Resolve an account by name to { name, portalId, key }.
+ * Reads the per-portal service key from $HUBSPOT_KEY_DIR/<portalId>.key.
+ * @param {string} name account name as keyed in accounts.json
+ * @returns {{ name: string, portalId: string, key: string }}
+ */
+export function account(name, cfg = fallbackConfig()) {
+  const accounts = loadAccounts(cfg);
+  const entry = accounts[name];
+  if (!entry || typeof entry !== 'object' || !entry.portalId) {
+    const known = Object.keys(accounts)
+      .filter((k) => !k.startsWith('_'))
+      .join(', ');
+    throw new Error(`Unknown account "${name}". Known accounts: ${known || '(none)'}`);
+  }
+  const portalId = String(entry.portalId);
+  const dir = keyDir(cfg);
+  const keyFile = join(dir, `${portalId}.key`);
+  if (!existsSync(keyFile)) {
+    throw new Error(
+      `No key for account "${name}" (portal ${portalId}) at ${keyFile}\n` +
+        `  Create it: printf '%s' 'pat-naX-...' > ${keyFile} && chmod 600 ${keyFile}\n` +
+        `  (override the directory with $HUBSPOT_KEY_DIR)`
+    );
+  }
+  const key = readFileSync(keyFile, 'utf8').trim();
+  if (!key) {
+    throw new Error(`Key file ${keyFile} for account "${name}" is empty.`);
+  }
+  return { name, portalId, key };
+}
+
+/**
+ * Make an authenticated JSON request against the HubSpot API.
+ * Never throws on a non-2xx status — returns { ok, status, json } so callers
+ * decide how to react (some endpoints, e.g. slug lookups, treat 404 as "none").
+ * @param {{ key: string }} acct account object from account()
+ * @param {string} method HTTP method
+ * @param {string} path API path beginning with '/'
+ * @param {*} [body] optional JSON body
+ * @returns {Promise<{ ok: boolean, status: number, json: any }>}
+ */
+export async function hub(acct, method, path, body) {
+  if (!acct || !acct.key) {
+    throw new Error('hub() requires an account object with a key (from account()).');
+  }
+  const res = await fetch(API + path, {
+    method,
+    headers: {
+      Authorization: `Bearer ${acct.key}`,
+      'Content-Type': 'application/json',
+    },
+    body: body !== undefined ? JSON.stringify(body) : undefined,
+  });
+  const text = await res.text();
+  let json;
+  try {
+    json = text ? JSON.parse(text) : {};
+  } catch {
+    json = { raw: text };
+  }
+  return { ok: res.ok, status: res.status, json };
+}
+
+/**
+ * Follow v3 paging (paging.next.after), accumulating results[].
+ * Throws with a clear message on any non-ok page.
+ * @param {{ key: string }} acct account object
+ * @param {string} path API path (query string allowed)
+ * @returns {Promise<any[]>} concatenated results
+ */
+export async function getAll(acct, path) {
+  const out = [];
+  let after;
+  do {
+    const sep = path.includes('?') ? '&' : '?';
+    const url = `${path}${sep}limit=100${after ? `&after=${after}` : ''}`;
+    const { ok, status, json } = await hub(acct, 'GET', url);
+    if (!ok) {
+      const msg = json?.message || json?.category || JSON.stringify(json).slice(0, 200);
+      throw new Error(`GET ${url} -> ${status}: ${msg}`);
+    }
+    out.push(...(json.results || []));
+    after = json.paging?.next?.after;
+  } while (after);
+  return out;
+}
+
+/**
+ * Pure: pick the page id whose slug matches `slug` from a v3 page list.
+ * '' (empty string) addresses the homepage. Exposed for unit testing.
+ * @param {any[]} pages array of page objects ({ id, slug })
+ * @param {string} slug normalized slug ('' = homepage)
+ * @returns {string|null} matched page id or null
+ */
+export function matchPageSlug(pages, slug) {
+  const want = slug == null ? '' : String(slug);
+  for (const p of pages) {
+    if (String(p.slug ?? '') === want) return String(p.id);
+  }
+  return null;
+}
+
+/**
+ * Resolve a CMS site page id by slug ('' = homepage). Read call; returns null
+ * when no page matches.
+ * @param {{ key: string }} acct account object
+ * @param {string} slug normalized slug
+ * @returns {Promise<string|null>}
+ */
+export async function resolvePageBySlug(acct, slug) {
+  const pages = await getAll(acct, '/cms/v3/pages/site-pages');
+  return matchPageSlug(pages, slug);
+}
+
+/**
+ * Pure: pick the legacy blog contentGroupId whose slug matches from the v2
+ * /content/api/v2/blogs `objects` array. Exposed for unit testing.
+ * @param {any[]} blogs array of blog containers ({ id, slug })
+ * @param {string} slug normalized blog slug
+ * @returns {string|null} matched contentGroupId or null
+ */
+export function matchBlogSlug(blogs, slug) {
+  const want = slug == null ? '' : String(slug);
+  for (const b of blogs) {
+    if (String(b.slug ?? '') === want) return String(b.id);
+  }
+  return null;
+}
+
+/**
+ * Resolve a blog container (contentGroupId) by slug via the legacy
+ * /content/api/v2/blogs endpoint. Returns null when no blog matches.
+ * Matching is by slug — never objects[0] — so a stale "Old" blog cannot win.
+ * @param {{ key: string }} acct account object
+ * @param {string} slug normalized blog slug
+ * @returns {Promise<string|null>}
+ */
+export async function resolveBlogBySlug(acct, slug) {
+  const { ok, status, json } = await hub(acct, 'GET', '/content/api/v2/blogs?limit=100');
+  if (!ok) {
+    const msg = json?.message || json?.category || JSON.stringify(json).slice(0, 200);
+    throw new Error(`GET /content/api/v2/blogs -> ${status}: ${msg}`);
+  }
+  return matchBlogSlug(json.objects || [], slug);
+}

package/src/lib/orchestrate.mjs

@@ -0,0 +1,141 @@
+// sync/lib/orchestrate.mjs — shared adapter loading + dependency ordering.
+//
+// The pull/push orchestrators (sync/pull.mjs, sync/push.mjs) both need to:
+//   1. discover every adapter under sync/adapters/*.mjs and key it by `name`, and
+//   2. run those adapters in DEPENDENCY ORDER (an adapter's `dependsOn` names must
+//      have run first).
+//
+// On PUSH the order is load-bearing: the `forms` and `assets` adapters POPULATE the
+// per-account registry (logical key -> target id / hosted url) that downstream
+// adapters (theme, pages, content, blog) RESOLVE via refs.resolve. Run a consumer
+// before its producer and resolve() hard-fails on an unmapped ref. topoSort encodes
+// that contract from each adapter's declared `dependsOn`, so neither orchestrator
+// hardcodes a sequence — add an adapter, declare its deps, and the order follows.
+//
+// On PULL order is not strictly required (pull is read-only and auto-registers refs),
+// but we run the SAME topo order for determinism and so a producer's registry entries
+// exist before a consumer pulls (e.g. forms register source GUIDs first).
+//
+// PURE except for loadAdapters' dynamic import: topoSort is a pure function over a
+// {name: {dependsOn}} map, so it unit-tests with fake adapter modules and no fs.
+
+import { readdir } from 'node:fs/promises';
+import { join, dirname } from 'node:path';
+import { fileURLToPath, pathToFileURL } from 'node:url';
+
+const __dirname = dirname(fileURLToPath(import.meta.url));
+// sync/lib/orchestrate.mjs -> sync/adapters/
+const ADAPTERS_DIR = join(__dirname, '..', 'adapters');
+
+/**
+ * loadAdapters(dir?) -> { [name]: module }
+ *
+ * Dynamically import every `*.mjs` under sync/adapters/ and key each module by its
+ * exported `name`. Each adapter module exports: name, dependsOn[], pull(), push().
+ * Throws if two adapters declare the same `name` (ambiguous graph) or a module is
+ * missing its `name`.
+ *
+ * @param {string} [dir] override the adapters directory (used by tests)
+ * @returns {Promise<Record<string, any>>}
+ */
+export async function loadAdapters(dir = ADAPTERS_DIR) {
+  const entries = await readdir(dir, { withFileTypes: true });
+  const files = entries
+    .filter((e) => e.isFile() && e.name.endsWith('.mjs'))
+    .map((e) => e.name)
+    .sort(); // deterministic import order
+
+  const adapters = {};
+  for (const file of files) {
+    const mod = await import(pathToFileURL(join(dir, file)).href);
+    const name = mod.name ?? mod.default?.name;
+    if (!name) {
+      throw new Error(`Adapter ${file} does not export a \`name\``);
+    }
+    if (adapters[name]) {
+      throw new Error(`Duplicate adapter name "${name}" (in ${file})`);
+    }
+    // Normalize to a single object carrying name/dependsOn/pull/push regardless of
+    // whether the adapter exported them individually or via `default`.
+    const def = mod.default ?? mod;
+    adapters[name] = {
+      name,
+      dependsOn: def.dependsOn ?? mod.dependsOn ?? [],
+      pull: def.pull ?? mod.pull,
+      push: def.push ?? mod.push,
+      module: mod,
+    };
+  }
+  return adapters;
+}
+
+/**
+ * topoSort(adapters) -> string[]
+ *
+ * Kahn/DFS topological sort of adapter NAMES by their `dependsOn`. A name appears
+ * AFTER every name it depends on. Deterministic: ties are broken alphabetically so a
+ * given graph always yields the same order.
+ *
+ * Throws on:
+ *   - an unknown dependency (a `dependsOn` entry with no matching adapter), and
+ *   - a dependency cycle (naming the offending nodes).
+ *
+ * `adapters` is any map of `name -> { dependsOn: string[] }` (the real adapter
+ * registry, or a fake one in tests).
+ *
+ * @param {Record<string, { dependsOn?: string[] }>} adapters
+ * @returns {string[]} adapter names in dependency order
+ */
+export function topoSort(adapters) {
+  const names = Object.keys(adapters).sort(); // stable, alphabetical tie-break
+
+  // Validate dependencies up front so the error names the missing dep, not a
+  // confusing "cycle" later.
+  for (const name of names) {
+    for (const dep of adapters[name].dependsOn ?? []) {
+      if (!(dep in adapters)) {
+        throw new Error(`Adapter "${name}" dependsOn unknown adapter "${dep}"`);
+      }
+    }
+  }
+
+  // Kahn's algorithm: repeatedly emit a node whose deps are all already emitted.
+  // Among the currently-ready nodes we pick the alphabetically-first, so roots come
+  // out first and the order is fully deterministic (e.g. assets, forms before their
+  // consumers). This keeps producers (forms/assets) ahead of consumers and reads
+  // naturally in the orchestrator's printed order.
+  const remaining = new Set(names);
+  const order = [];
+
+  while (remaining.size) {
+    const ready = [...remaining]
+      .filter((name) => (adapters[name].dependsOn ?? []).every((dep) => !remaining.has(dep)))
+      .sort();
+    if (ready.length === 0) {
+      // Everything left is part of (or blocked by) a cycle. Report the offenders.
+      const cycle = describeCycle(adapters, remaining);
+      throw new Error(`Dependency cycle detected: ${cycle}`);
+    }
+    const next = ready[0];
+    order.push(next);
+    remaining.delete(next);
+  }
+
+  return order;
+}
+
+// Walk dependency edges among the still-unresolved nodes to surface a concrete cycle
+// chain (a -> b -> c -> a) for the error message.
+function describeCycle(adapters, remaining) {
+  const start = [...remaining].sort()[0];
+  const path = [];
+  const seen = new Set();
+  let node = start;
+  while (node != null && !seen.has(node)) {
+    seen.add(node);
+    path.push(node);
+    node = (adapters[node].dependsOn ?? []).find((dep) => remaining.has(dep));
+  }
+  if (node != null) path.push(node); // close the loop back to a repeated node
+  return path.join(' -> ');
+}