@owox/connectors 0.36.0 → 0.37.0

package/dist/index.js

@@ -349,6 +349,7 @@ var require_index = __commonJS({
       async run() {
         var _a;
         this.context.emit(new ControlEvent(CONTROL_ACTION.STARTED));
+        this._rowsByNode = /* @__PURE__ */ new Map();
         try {
           this.context.validate();
           this._processRunConfig();
@@ -375,6 +376,7 @@ var require_index = __commonJS({
             }
           }
           this._reportAccountOutcomes(state);
+          this._reportEmptyNodes([...plainNodes, ...timeSeriesNodes]);
           this.source.onImportComplete(this.context);
           this.context.emit(new ControlEvent(CONTROL_ACTION.COMPLETED));
         } catch (error) {
@@ -510,10 +512,11 @@ var require_index = __commonJS({
        * @param {object} state run state from _createRunState
        * @param {object|null} account the account this unit belongs to
        * @param {Function} work async thunk performing the fetch + write
+       * @param {string} subject what the unit imports -- a node name, or the day for day-by-day
        * @returns {Promise<boolean>} true when the unit completed
        * @private
        */
-      async _runForAccount(state, account, work) {
+      async _runForAccount(state, account, work, subject) {
         try {
           await work();
           state.succeeded.add(this._accountKey(account));
@@ -521,7 +524,7 @@ var require_index = __commonJS({
         } catch (error) {
           this.source.onAccountError(account, error);
           if (error == null ? void 0 : error[STORAGE_FAILURE]) throw error;
-          this._recordAccountFailure(state, account, error);
+          this._recordAccountFailure(state, account, error, subject);
           return false;
         }
       }
@@ -560,23 +563,29 @@ var require_index = __commonJS({
        * @param {object} state run state from _createRunState
        * @param {object|null} account the account that failed
        * @param {Error} error the failure to classify
+       * @param {string} [subject] what was being imported, named in the report as main did
        * @private
        */
-      _recordAccountFailure(state, account, error) {
+      _recordAccountFailure(state, account, error, subject) {
         const accountId = this._accountKey(account);
         const isSkip = (error == null ? void 0 : error.isWarning) === true;
         if (!isSkip) state.passHardFailures += 1;
         let entry = state.issues.get(accountId);
         if (!entry) {
-          entry = { errors: [], reported: /* @__PURE__ */ new Set() };
+          entry = { errors: [], reported: /* @__PURE__ */ new Set(), subjects: /* @__PURE__ */ new Set() };
           state.issues.set(accountId, entry);
         }
         entry.errors.push(error);
+        if (subject) entry.subjects.add(subject);
         const level = isSkip ? LOG_LEVEL.WARN : LOG_LEVEL.ERROR;
         if (entry.reported.has(error.message)) return;
         entry.reported.add(error.message);
         const what = account === null || account === void 0 ? isSkip ? "Skipped" : "Import failed" : `${isSkip ? "Skipped account" : "Error processing account"} ${accountId}`;
-        this.context.log(level, `${what}: ${error.message}`);
+        const line = `${what}: ${error.message}`;
+        this.context.log(
+          level,
+          subject ? `Importing ${subject}: ${line[0].toLowerCase()}${line.slice(1)}` : line
+        );
       }
       /**
        * Whether the account has failed outright (not merely been skipped) with the same error
@@ -635,7 +644,48 @@ var require_index = __commonJS({
        */
       _emitCursor(state, date) {
         if (state.cursorHalted) return;
-        this.context.emit(new StateEvent({ lastRequestedDate: date }));
+        const claimed = state.cursorCap && date > state.cursorCap ? state.cursorCap : date;
+        if (claimed !== date && claimed === state.lastClaimed) return;
+        state.lastClaimed = claimed;
+        this.context.emit(new StateEvent({ lastRequestedDate: claimed }));
+      }
+      /**
+       * The last day a node is asked for: the run's end, or `endLagDays` before today for an API
+       * that reports only completed days, whichever is earlier.
+       * @private
+       */
+      _nodeWindowEnd(node, dateRange) {
+        var _a, _b;
+        const lag = ((_b = (_a = this.source).getEndLagDays) == null ? void 0 : _b.call(_a, node.name)) ?? 0;
+        if (!lag) return dateRange.endDate;
+        const today = this._parseDate(this._formatDate(/* @__PURE__ */ new Date()));
+        const lagged = this._formatDate(new Date(today - lag * 864e5));
+        return lagged < dateRange.endDate ? lagged : dateRange.endDate;
+      }
+      /**
+       * Notes where `endLagDays` stopped a manual backfill short of its EndDate. The run completes
+       * either way, and nothing else on it names the days it left out.
+       * @private
+       */
+      _logLagCut(node, dateRange, end) {
+        var _a;
+        if (((_a = this.context.runConfig) == null ? void 0 : _a.type) !== RUN_CONFIG_TYPE.MANUAL_BACKFILL) return;
+        if (end >= dateRange.endDate) return;
+        this.context.log(
+          LOG_LEVEL.INFO,
+          `Node "${node.name}" stops at ${end}, not at the EndDate ${dateRange.endDate}: endLagDays ends its window that many days before today`
+        );
+      }
+      /**
+       * Notes a node that has nothing to ask for this run because its lag ends its window
+       * before the run's first day.
+       * @private
+       */
+      _logLaggedOut(node, dateRange, end) {
+        this.context.log(
+          LOG_LEVEL.INFO,
+          `Node "${node.name}" asks for nothing this run: endLagDays ends its window on ${end}, before the run's first day ${dateRange.startDate}`
+        );
       }
       /**
        * Lists each account's latest error; for a source without accounts, just the error.
@@ -725,7 +775,9 @@ var require_index = __commonJS({
           } else if (state.accountless) {
             message = `Nothing was imported because access was refused: ${describe(entries)}`;
           } else {
-            message = `All ${state.attemptedCount} accounts were skipped, so nothing was imported. This points to a global failure, such as an expired access token, rather than individual accounts being inaccessible. Errors: ${describe(entries)}`;
+            const subjects = new Set(entries.flatMap(([, entry]) => [...entry.subjects]));
+            const where = subjects.size === 1 ? ` while importing ${[...subjects][0]}` : "";
+            message = `All ${state.attemptedCount} accounts were skipped${where}, so nothing was imported. This points to a global failure, such as an expired access token, rather than individual accounts being inaccessible. Errors: ${describe(entries)}`;
           }
           const error = new Error(message);
           error.isWarning = failed.length === 0;
@@ -808,11 +860,13 @@ var require_index = __commonJS({
         } catch (error) {
           throw asStorageFailure(error);
         }
+        this._countRows(writer.nodeName, rows.length);
       }
       /**
        * Resolves short links for the fields a schema node declares under `shortLinks`, writing
-       * each landing page next to its original. No-op when the node has no spec, Process Short
-       * Links is off, or no spec has its field (and its `_parsed` target) selected.
+       * the address each short link service points to next to the original. No-op when the node
+       * has no spec, Process Short Links is off, or no spec has its field (and its `_parsed`
+       * target) selected.
        *
        * The helpers are bare globals of Core/Utils/ShortLinksUtils.js, a script in the bundle's
        * scope; nothing here touches them for a node without a spec.
@@ -823,7 +877,7 @@ var require_index = __commonJS({
        * @returns {Promise<object[]>} records with resolved links, or the same array
        */
       async resolveShortLinks(nodeName, data, fields) {
-        var _a, _b, _c, _d, _e;
+        var _a, _b, _c, _d;
         const specs = (_c = (_b = (_a = this.source) == null ? void 0 : _a.fieldsSchema) == null ? void 0 : _b[nodeName]) == null ? void 0 : _c.shortLinks;
         if (!Array.isArray(specs) || specs.length === 0 || !data.length) return data;
         if (((_d = this.context.getParameter("ProcessShortLinks")) == null ? void 0 : _d.value) === false) return data;
@@ -834,12 +888,7 @@ var require_index = __commonJS({
         if (activeSpecs.length === 0) return data;
         const cache = this._shortLinksCache();
         return resolveShortLinkFields(data, activeSpecs, {
-          nestedPathHosts: [
-            ...getShortLinkDomainsFromEnv(),
-            // Domains saved by the former Facebook "Short Link Domains" setting keep working
-            // until every environment sets CONNECTOR_SHORT_LINK_DOMAINS.
-            ...parseShortLinkDomains((_e = this.context.getParameter("ShortLinkDomains")) == null ? void 0 : _e.value)
-          ],
+          allowedHosts: getShortLinkDomainsFromEnv(),
           resolvedLinksCache: cache.resolved,
           failedLinks: cache.failed
         });
@@ -918,7 +967,7 @@ var require_index = __commonJS({
               onBatch: (batch) => this._writeBatch(writer, batch, fields)
             });
             await this._writeBatch(writer, data, fields);
-          });
+          }, nodeName);
         }
       }
       /**
@@ -943,6 +992,16 @@ var require_index = __commonJS({
        */
       async _processTimeSeriesNodes(nodes, accounts, state) {
         if (!nodes.length) return;
+        const lagging = nodes.filter(
+          (node) => {
+            var _a, _b;
+            return this.source.getDateStrategy(node.name) !== DATE_STRATEGY.NONE && (((_b = (_a = this.source).getEndLagDays) == null ? void 0 : _b.call(_a, node.name)) ?? 0) > 0;
+          }
+        );
+        const dateRange = lagging.length ? this.getDateRange() : null;
+        if (dateRange) {
+          state.cursorCap = lagging.map((node) => this._nodeWindowEnd(node, dateRange)).reduce((earliest, date) => date < earliest ? date : earliest);
+        }
         const dayByDayNodes = [];
         const windowEndDates = [];
         for (const node of nodes) {
@@ -981,6 +1040,12 @@ var require_index = __commonJS({
           this.context.log(LOG_LEVEL.WARN, `Could not determine date range for node "${node.name}"`);
           return null;
         }
+        const endDate = this._nodeWindowEnd(node, dateRange);
+        if (endDate < dateRange.startDate) {
+          this._logLaggedOut(node, dateRange, endDate);
+          return null;
+        }
+        this._logLagCut(node, dateRange, endDate);
         const writers = /* @__PURE__ */ new Map();
         let completedBy = 0;
         this._beginPass(state);
@@ -992,17 +1057,17 @@ var require_index = __commonJS({
               fields: node.fields,
               accountId: (account == null ? void 0 : account.id) ?? null,
               startDate: dateRange.startDate,
-              endDate: dateRange.endDate
+              endDate
             });
             await this._writeBatch(writer, data, node.fields);
-          });
+          }, node.name);
           if (done) completedBy += 1;
         }
         if (completedBy === 0 || state.passHardFailures > 0) {
           state.cursorHalted = true;
           return null;
         }
-        return dateRange.endDate;
+        return endDate;
       }
       /**
        * DATE_STRATEGY.NONE: the node carries no date window at all, so there is
@@ -1042,8 +1107,15 @@ var require_index = __commonJS({
           return;
         }
         const writers = /* @__PURE__ */ new Map();
-        for (const date of this._iterateDates(dateRange.startDate, dateRange.endDate)) {
+        const endDates = new Map(nodes.map((node) => [node, this._nodeWindowEnd(node, dateRange)]));
+        for (const [node, end] of endDates) {
+          if (end < dateRange.startDate) this._logLaggedOut(node, dateRange, end);
+          else this._logLagCut(node, dateRange, end);
+        }
+        const lastDay = [...endDates.values()].reduce((latest, date) => date > latest ? date : latest);
+        for (const date of this._iterateDates(dateRange.startDate, lastDay)) {
           const formattedDate = this._formatDate(date);
+          const dayNodes = nodes.filter((node) => endDates.get(node) >= formattedDate);
           let completedBy = 0;
           this._beginPass(state);
           for (const account of accounts) {
@@ -1052,7 +1124,7 @@ var require_index = __commonJS({
               continue;
             }
             const done = await this._runForAccount(state, account, async () => {
-              for (const node of nodes) {
+              for (const node of dayNodes) {
                 const writer = this._nodeWriter(writers, node);
                 const data = await this.source.fetchData({
                   nodeName: node.name,
@@ -1063,7 +1135,7 @@ var require_index = __commonJS({
                 });
                 await this._writeBatch(writer, data, node.fields);
               }
-            });
+            }, formattedDate);
             if (done) completedBy += 1;
           }
           if (completedBy === 0 && state.passHardFailures === 0 && accounts.length > 0) {
@@ -1123,7 +1195,7 @@ var require_index = __commonJS({
               endDate: null
             });
             if (data && data.length) batches.push(data);
-          });
+          }, nodeName);
           if (!done) missing.add(this._accountKey(account));
         }
         if (missing.size) {
@@ -1171,6 +1243,29 @@ var require_index = __commonJS({
           );
         }
         await storage.replaceData(snapshot);
+        this._countRows(nodeName, snapshot.length);
+      }
+      /** @private */
+      _countRows(nodeName, count) {
+        if (!this._rowsByNode) return;
+        this._rowsByNode.set(nodeName, (this._rowsByNode.get(nodeName) ?? 0) + count);
+      }
+      /**
+       * Reports 0 rows written for every node that loaded nothing in a completed run. The
+       * storages report only the rows they write, so without this Run History showed such a run
+       * as a plain success with no count at all.
+       *
+       * @param {object[]} nodes the run's planned nodes
+       * @private
+       */
+      _reportEmptyNodes(nodes) {
+        var _a;
+        for (const node of nodes) {
+          if (((_a = this._rowsByNode) == null ? void 0 : _a.get(node.name)) > 0) continue;
+          this.context.emitAnalytics("rows_written", 0, {
+            node: this.source.getDestinationName(node.name, node.schema)
+          });
+        }
       }
       /**
        * One row per unique key, the last one read: what a MERGE keeps. The storages dedupe
@@ -1776,6 +1871,13 @@ var require_index = __commonJS({
       getDateStrategy(nodeName) {
         return DATE_STRATEGY.DAY_BY_DAY;
       }
+      /**
+       * How many days before today a time-series node's window ends, for an API that reports
+       * only completed days. 0: the window ends today.
+       */
+      getEndLagDays(nodeName) {
+        return 0;
+      }
       /**
        * Called after all nodes for an account are processed.
        */
@@ -1908,14 +2010,19 @@ var require_index = __commonJS({
             if (!response.ok) {
               const bodyText = typeof response.text === "function" ? await response.text().catch(() => "") : "";
               const snippet = bodyText ? ` — ${bodyText.slice(0, 300)}` : "";
-              const error = new Error(`HTTP ${response.status}: ${response.statusText}${snippet}`);
-              error.response = response;
-              error.statusCode = response.status;
-              error.responseBody = bodyText;
+              let payload;
               try {
-                error.payload = JSON.parse(bodyText);
+                payload = JSON.parse(bodyText);
               } catch {
               }
+              const providerMessage = _AbstractSource._providerErrorMessage(payload);
+              const error = new Error(
+                providerMessage ? `HTTP ${response.status}: ${providerMessage}` : `HTTP ${response.status}: ${response.statusText}${snippet}`
+              );
+              error.response = response;
+              error.statusCode = response.status;
+              error.responseBody = bodyText;
+              if (payload !== void 0) error.payload = payload;
               if (await this.isValidToRetry(error) && attempt < totalAttempts - 1) {
                 const delay = this.calculateBackoff(attempt, initialDelay);
                 this.context.log(
@@ -1954,6 +2061,28 @@ var require_index = __commonJS({
           }
         }
       }
+      /**
+       * The reason a provider gave in its JSON error body, looked up where main's
+       * _extractErrorInfo looked for it, and in a JSON:API error's `detail`. Not its `title`: that
+       * is the same summary for every occurrence, and the body snippet it would replace names the
+       * field in `source.pointer`.
+       *
+       * @param {*} payload - the parsed error body
+       * @returns {string|null} null when the body names no message
+       */
+      static _providerErrorMessage(payload) {
+        var _a;
+        const firstError = Array.isArray(payload == null ? void 0 : payload.errors) ? payload.errors[0] : void 0;
+        const candidates = [
+          (_a = payload == null ? void 0 : payload.error) == null ? void 0 : _a.message,
+          payload == null ? void 0 : payload.message,
+          payload == null ? void 0 : payload.errorMessage,
+          payload == null ? void 0 : payload.error_message,
+          firstError == null ? void 0 : firstError.message,
+          firstError == null ? void 0 : firstError.detail
+        ];
+        return candidates.find((message) => typeof message === "string" && message.trim()) ?? null;
+      }
       /**
        * The fetch options to use for the next redirect hop.
        *
@@ -2321,6 +2450,19 @@ var require_index = __commonJS({
         return fieldsParam.value.split(",").map((field) => field.trim()).filter((field) => field !== "").map((field) => field.split(" ")).filter((field) => field.length === 2).map((field) => field[1]);
       }
       //----------------------------------------------------------------
+      //---- missingSelectedFieldError ----------------------------------
+      /**
+       * The error for a field the Data Mart selects that the connector's schema does not have.
+       * Its usual cause is a connector whose fields changed after the Data Mart was set up.
+       * @param {string} columnName
+       * @returns {Error}
+       */
+      missingSelectedFieldError(columnName) {
+        return new Error(
+          `Field "${columnName}" is selected for import, but the connector does not provide it. If the connector's fields changed after this Data Mart was set up, open Edit Fields in the Data Mart's Input Source and save: a field the connector no longer provides is dropped.`
+        );
+      }
+      //----------------------------------------------------------------
       //---- getColumnType -----------------------------------------------
       /**
        * Get column type for storage from schema
@@ -2817,6 +2959,7 @@ var require_index = __commonJS({
         this._activeErrorHandler = null;
         this._pendingBackoff = null;
         this._tokenCache = /* @__PURE__ */ new Map();
+        this._extractedByNode = /* @__PURE__ */ new Map();
       }
       _compileNodes(nodes = {}) {
         const schema = {};
@@ -2867,8 +3010,13 @@ var require_index = __commonJS({
       getDateStrategy(nodeName) {
         return nodeDateStrategy(this.model.nodes[nodeName]);
       }
+      getEndLagDays(nodeName) {
+        const node = this.model.nodes[nodeName];
+        if (nodeDateStrategy(node) === DATE_STRATEGY.NONE) return 0;
+        return node.incremental.endLagDays ?? 0;
+      }
       async fetchData({ nodeName, fields, accountId, startDate, endDate }) {
-        var _a, _b, _c, _d;
+        var _a, _b, _c, _d, _e;
         const node = this.model.nodes[nodeName];
         if (!node) throw new Error(`DeclarativeSource: unknown node "${nodeName}"`);
         const scope = this._scope({ accountId, startDate, endDate, fields });
@@ -2936,8 +3084,12 @@ var require_index = __commonJS({
           this.runLimits
         );
         const rawRecords = await retriever.run(scope);
+        const nodeTag = ((_d = this.context.getParameter("DestinationTableName")) == null ? void 0 : _d.value) ?? nodeName;
+        this._extractedByNode.set(nodeName, {
+          tag: nodeTag,
+          count: (((_e = this._extractedByNode.get(nodeName)) == null ? void 0 : _e.count) ?? 0) + rawRecords.length
+        });
         if (rawRecords.length > 0) {
-          const nodeTag = ((_d = this.context.getParameter("DestinationTableName")) == null ? void 0 : _d.value) ?? nodeName;
           this.context.emitAnalytics("rows_extracted", rawRecords.length, { node: nodeTag });
         }
         const recordFilter = node.recordFilter ? new RecordFilter(node.recordFilter) : null;
@@ -2952,6 +3104,37 @@ var require_index = __commonJS({
         }
         return this._selectFields(new FieldCaster(node.fields || {}).cast(records), node, fields);
       }
+      /**
+       * Names each node whose requests got no records in the whole run. A record path that
+       * misses the data or a wrong parameter value completes like a run with nothing new, so this
+       * is the only hint the run leaves. INFO, not WARN: an API with nothing for the window is
+       * also a normal outcome.
+       */
+      onImportComplete() {
+        var _a;
+        for (const [nodeName, { tag, count }] of this._extractedByNode) {
+          if (count > 0) continue;
+          this.context.emitAnalytics("rows_extracted", 0, { node: tag });
+          this.context.log(
+            LOG_LEVEL.INFO,
+            `No records came back for node "${nodeName}" in this run. If you expected data, check the node's record path, and the parameters and dates its request uses.` + DeclarativeSource._partitionHint((_a = this.model.nodes[nodeName]) == null ? void 0 : _a.partitionRouter)
+          );
+        }
+      }
+      /**
+       * A partitioned node sends one request per slice, and none when there are no slices, so
+       * its own request is not where an empty run starts. Named as the builder's Partition form
+       * names the fields.
+       */
+      static _partitionHint(partitionRouter) {
+        if ((partitionRouter == null ? void 0 : partitionRouter.type) === "substream") {
+          return " Its requests run once per parent record, so also check Parent record path and Parent key: when they find nothing, the node sends no requests.";
+        }
+        if ((partitionRouter == null ? void 0 : partitionRouter.type) === "list") {
+          return " Its requests run once per value of its list, so also check Values or Values from parameter: an empty list sends no requests.";
+        }
+        return "";
+      }
       /**
        * Keeps the fields the Data Mart selected, and the node's unique keys, which the storage
        * merges on. A bundled source reads only the selected fields; casting every declared one
@@ -2982,9 +3165,10 @@ var require_index = __commonJS({
       async isValidToRetry(error) {
         const handler = this._activeErrorHandler;
         if (!handler) {
-          this._pendingBackoff = null;
           error._declAction = null;
-          return this._defaultRetryable(error);
+          const retry2 = this._defaultRetryable(error);
+          this._pendingBackoff = retry2 ? { filter: null, response: error.response } : null;
+          return retry2;
         }
         let bodyText = "";
         let bodyJson = null;
@@ -3003,13 +3187,23 @@ var require_index = __commonJS({
         return retry;
       }
       calculateBackoff(attempt, initialDelay) {
-        if (this._pendingBackoff && this._activeErrorHandler) {
-          const { filter, response } = this._pendingBackoff;
-          this._pendingBackoff = null;
-          const ms = this._activeErrorHandler.delayMs(filter, response, attempt, initialDelay);
+        const pending = this._pendingBackoff;
+        this._pendingBackoff = null;
+        if (pending && this._activeErrorHandler) {
+          const ms = this._activeErrorHandler.delayMs(
+            pending.filter,
+            pending.response,
+            attempt,
+            initialDelay
+          );
           if (ms != null) return ms;
         }
-        return super.calculateBackoff(attempt, initialDelay);
+        const fallback = super.calculateBackoff(attempt, initialDelay);
+        if (pending && !process.env.OW_TEST) {
+          const ms = retryAfterDelayMs(pending.response);
+          if (ms != null) return Math.max(ms, fallback);
+        }
+        return fallback;
       }
       /**
        * A 401 or 403 skips the account as refused access, but not one a filter mapped to FAIL:
@@ -3220,6 +3414,16 @@ var require_index = __commonJS({
       if (!Number.isFinite(ms)) return null;
       return Math.min(Math.max(0, ms), MAX_HEADER_RETRY_DELAY_MS);
     }
+    function retryAfterDelayMs(response, headerName = "Retry-After") {
+      var _a, _b;
+      const raw = (_b = (_a = response == null ? void 0 : response.headers) == null ? void 0 : _a.get) == null ? void 0 : _b.call(_a, headerName);
+      if (raw == null) return null;
+      const secs = Number(raw);
+      if (Number.isFinite(secs)) return clampHeaderDelay(secs * 1e3);
+      const dateMs = Date.parse(raw);
+      if (!Number.isNaN(dateMs)) return clampHeaderDelay(dateMs - Date.now());
+      return null;
+    }
     class ErrorHandler {
       /**
        * @param {{ responseFilters?: Array<object>, backoff?: object }} config
@@ -3290,16 +3494,7 @@ var require_index = __commonJS({
         }
       }
       _waitTimeFromHeader(spec, response) {
-        var _a, _b;
-        if (!response) return null;
-        const headerName = spec && spec.header || "Retry-After";
-        const raw = (_b = (_a = response.headers) == null ? void 0 : _a.get) == null ? void 0 : _b.call(_a, headerName);
-        if (raw == null) return null;
-        const secs = Number(raw);
-        if (Number.isFinite(secs)) return clampHeaderDelay(secs * 1e3);
-        const dateMs = Date.parse(raw);
-        if (!Number.isNaN(dateMs)) return clampHeaderDelay(dateMs - Date.now());
-        return null;
+        return retryAfterDelayMs(response, spec && spec.header || "Retry-After");
       }
       _waitUntilTimeFromHeader(spec, response) {
         var _a, _b;
@@ -3497,6 +3692,45 @@ var require_index = __commonJS({
     const PAGINATION_TYPES = /* @__PURE__ */ new Set(["none", "offset", "page", "cursor"]);
     const INJECT_TARGETS = /* @__PURE__ */ new Set(["query", "header", "body", "path"]);
     const RESPONSE_FORMATS = /* @__PURE__ */ new Set(["json", "csv", "jsonl"]);
+    const MAX_END_LAG_DAYS = 30;
+    const DATE_WINDOW_PLACEHOLDER = /\{\{\s*(dateWindow(?:\.[a-zA-Z0-9_]+)*)\s*\}\}/;
+    function dateWindowTemplateIn(value) {
+      var _a;
+      if (typeof value === "string") return ((_a = DATE_WINDOW_PLACEHOLDER.exec(value)) == null ? void 0 : _a[1]) ?? null;
+      if (value && typeof value === "object") {
+        for (const item of Object.values(value)) {
+          const path = dateWindowTemplateIn(item);
+          if (path) return path;
+        }
+      }
+      return null;
+    }
+    function assertNoStrictDateWindowTemplate(nodeName, node) {
+      var _a, _b, _c;
+      const isAsync = ((_a = node.retriever) == null ? void 0 : _a.type) === "async";
+      for (const [where, spec] of [
+        // An async node sends its retriever's requests, never `request`.
+        ["request", isAsync ? void 0 : node.request],
+        ["retriever.submit", isAsync ? node.retriever.submit : void 0],
+        ["retriever.poll", isAsync ? node.retriever.poll : void 0],
+        ["partitionRouter.parent.request", (_c = (_b = node.partitionRouter) == null ? void 0 : _b.parent) == null ? void 0 : _c.request]
+      ]) {
+        if (!spec || typeof spec !== "object") continue;
+        const sendsBody = String(spec.method || "GET").toUpperCase() !== "GET";
+        for (const [place, value] of [
+          [`${where}.path`, spec.path],
+          [`${where}.body`, sendsBody ? spec.body : void 0]
+        ]) {
+          const path = dateWindowTemplateIn(value);
+          if (path) {
+            const fix = node.isFullRefresh ? 'Remove "isFullRefresh" and add an "incremental" block to fetch by date window, or remove the template.' : 'Add an "incremental" block to the node, or remove the template.';
+            throw new Error(
+              `ManifestParser: node "${nodeName}" uses "{{ ${path} }}" in ${place}, but the node is never given a date window: only a node with an "incremental" strategy, and without "isFullRefresh", is fetched one window at a time. ${fix}`
+            );
+          }
+        }
+      }
+    }
     const LEGACY_INJECT_PARAM = {
       cursor: "cursorParam",
       offset: "offsetParam",
@@ -3620,10 +3854,12 @@ var require_index = __commonJS({
     class ManifestParser {
       /**
        * @param {string} json
+       * @param {{ authoring?: boolean }} [options] - `authoring` adds the checks for a manifest
+       *   being published or tested, which a stored version must not start failing on.
        * @returns {object} ManifestModel
        */
-      parse(json) {
-        var _a, _b, _c, _d, _e, _f, _g, _h, _i, _j, _k, _l, _m, _n, _o, _p, _q, _r, _s, _t, _u, _v, _w, _x, _y, _z;
+      parse(json, { authoring = false } = {}) {
+        var _a, _b, _c, _d, _e, _f, _g, _h, _i, _j, _k, _l, _m, _n, _o, _p, _q, _r, _s, _t, _u, _v, _w, _x, _y, _z, _A;
         let raw;
         try {
           raw = JSON.parse(json);
@@ -3799,11 +4035,27 @@ var require_index = __commonJS({
               `ManifestParser: node "${nodeName}" has unknown incremental.strategy "${strategy}"`
             );
           }
+          const endLagDays = (_w = node.incremental) == null ? void 0 : _w.endLagDays;
+          if (endLagDays !== void 0) {
+            if (!Number.isInteger(endLagDays) || endLagDays < 0 || endLagDays > MAX_END_LAG_DAYS) {
+              throw new Error(
+                `ManifestParser: node "${nodeName}" incremental.endLagDays must be a whole number of days from 0 to ${MAX_END_LAG_DAYS}, e.g. 1 to leave out today`
+              );
+            }
+            if (strategy === "none") {
+              throw new Error(
+                `ManifestParser: node "${nodeName}" incremental.endLagDays needs a "day-by-day" or "range" strategy — a node with no date window has no window end to move`
+              );
+            }
+          }
           if (declaresDateWindow(node) && node.isFullRefresh) {
             throw new Error(
               `ManifestParser: node "${nodeName}" declares both "incremental" and "isFullRefresh", which are mutually exclusive — a full-refresh node replaces the whole table and is never given a date window, so the incremental block would be silently ignored. Remove "isFullRefresh" to fetch by date window, or remove the "incremental" block.`
             );
           }
+          if (authoring && !declaresDateWindow(node)) {
+            assertNoStrictDateWindowTemplate(nodeName, node);
+          }
           if (node.transformations !== void 0) {
             if (!Array.isArray(node.transformations)) {
               throw new Error(`ManifestParser: node "${nodeName}" transformations must be an array`);
@@ -3899,7 +4151,7 @@ var require_index = __commonJS({
               }
             }
             if (eh.backoff !== void 0) validateBackoff(eh.backoff, nodeName, "errorHandler");
-            if (((_w = node.retriever) == null ? void 0 : _w.type) === "async") {
+            if (((_x = node.retriever) == null ? void 0 : _x.type) === "async") {
               throw new Error(
                 `ManifestParser: node "${nodeName}" errorHandler is not supported with an async retriever — an async node paces itself through "retriever.poll.backoff" instead, so these filters and this backoff would never run. Remove "errorHandler", or make the node sync.`
               );
@@ -3952,7 +4204,7 @@ var require_index = __commonJS({
                 );
               }
             }
-            if (((_x = node.retriever) == null ? void 0 : _x.type) === "async") {
+            if (((_y = node.retriever) == null ? void 0 : _y.type) === "async") {
               throw new Error(
                 `ManifestParser: node "${nodeName}" partitionRouter is not supported with an async retriever`
               );
@@ -3998,7 +4250,7 @@ var require_index = __commonJS({
               }
             }
           }
-          const rType = ((_y = node.retriever) == null ? void 0 : _y.type) ?? "sync";
+          const rType = ((_z = node.retriever) == null ? void 0 : _z.type) ?? "sync";
           if (rType === "sync") {
             if (!node.request) {
               throw new Error(`ManifestParser: node "${nodeName}" is missing "request"`);
@@ -4130,7 +4382,7 @@ var require_index = __commonJS({
               );
             }
             for (const key of ["maxAttempts", "initialMs", "maxMs"]) {
-              const value = (_z = r.poll.backoff) == null ? void 0 : _z[key];
+              const value = (_A = r.poll.backoff) == null ? void 0 : _A[key];
               if (value === void 0) continue;
               const valid = typeof value === "number" && value > 0 && (key === "maxAttempts" ? Number.isInteger(value) : Number.isFinite(value));
               if (!valid) {
@@ -4209,7 +4461,7 @@ var require_index = __commonJS({
         const identifier = /^[A-Za-z_][A-Za-z0-9_]*$/;
         if (!/^[A-Za-z][A-Za-z0-9_]*$/.test(nodeName)) {
           throw new Error(
-            `ManifestParser: node "${nodeName}" must start with a letter and contain only letters, digits and underscores`
+            `ManifestParser: node "${nodeName}" must start with a letter and contain only letters, digits and underscores` + (nodeName.includes("/") ? `; the API path goes in the node's request.path` : "")
           );
         }
         if (node.destinationName !== void 0 && !identifier.test(String(node.destinationName))) {
@@ -5029,6 +5281,9 @@ var require_index = __commonJS({
     ];
     const CONCLUSIVE_DNS_FAILURES = /* @__PURE__ */ new Set(["ENOTFOUND", "ENODATA"]);
     const INCONCLUSIVE_LOOKUP_ATTEMPTS = 3;
+    function refusal(message) {
+      return Object.assign(new Error(message), { isRefusal: true });
+    }
     class SsrfGuard {
       /**
        * True if `ip` is a private/loopback/link-local address. Accepts both the
@@ -5138,11 +5393,11 @@ var require_index = __commonJS({
         }
         if (SsrfGuard._allowLocalEgress()) return;
         if (url.protocol !== "https:") {
-          throw new Error(`SsrfGuard: URL must use https, got "${url.protocol}"`);
+          throw refusal(`SsrfGuard: URL must use https, got "${url.protocol}"`);
         }
         const host = url.hostname.toLowerCase().replace(/\.$/, "");
         if (this._isBlockedLiteralHost(host)) {
-          throw new Error(`SsrfGuard: blocked IP/host "${host}"`);
+          throw refusal(`SsrfGuard: blocked IP/host "${host}"`);
         }
         await this._assertResolvesToPublic(host);
       }
@@ -5192,7 +5447,7 @@ var require_index = __commonJS({
         for (const a of list) {
           const address = a && typeof a === "object" ? a.address : a;
           if (SsrfGuard.isBlockedIp(address)) {
-            throw new Error(`SsrfGuard: host "${host}" resolves to blocked address "${address}"`);
+            throw refusal(`SsrfGuard: host "${host}" resolves to blocked address "${address}"`);
           }
         }
       }
@@ -6626,25 +6881,28 @@ var require_index = __commonJS({
         return base64.replace(/\+/g, "-").replace(/\//g, "_").replace(/=/g, "");
       }
     };
-    async function resolveShortLinkFields(data, specs, { nestedPathHosts = [], resolvedLinksCache = /* @__PURE__ */ new Map(), failedLinks = /* @__PURE__ */ new Set() } = {}) {
+    async function resolveShortLinkFields(data, specs, { allowedHosts = [], resolvedLinksCache = /* @__PURE__ */ new Map(), failedLinks = /* @__PURE__ */ new Set() } = {}) {
       if (!Array.isArray(data) || data.length === 0 || !Array.isArray(specs) || specs.length === 0) return data;
-      const candidates = _collectCandidateUrls(data, specs, nestedPathHosts);
+      const candidates = _collectCandidateUrls(data, specs, allowedHosts);
       const hasPlainTargets = specs.some((spec) => !spec.urlKey);
       if (candidates.length === 0 && !hasPlainTargets) return data;
       const uncachedLinks = candidates.filter((url) => !resolvedLinksCache.has(url)).map((originalUrl) => ({ originalUrl }));
-      const freshlyResolved = await _resolveShortLinks(uncachedLinks);
+      const freshlyResolved = await _resolveShortLinks(uncachedLinks, allowedHosts);
       freshlyResolved.forEach((link) => {
         resolvedLinksCache.set(link.originalUrl, link.resolvedUrl);
         if (!link.ok) failedLinks.add(link.originalUrl);
       });
-      return data.map((record) => specs.reduce((current, spec) => _applySpec(current, spec, resolvedLinksCache), record));
+      const resolvable = new Set(candidates);
+      return data.map(
+        (record) => specs.reduce((current, spec) => _applySpec(current, spec, resolvedLinksCache, resolvable), record)
+      );
     }
-    function _collectCandidateUrls(data, specs, nestedPathHosts) {
+    function _collectCandidateUrls(data, specs, allowedHosts) {
       const unique = /* @__PURE__ */ new Set();
       data.forEach((record) => {
         specs.forEach((spec) => {
           _readUrls(record[spec.field], spec).forEach((url) => {
-            if (_isPotentialShortLink(url, nestedPathHosts)) unique.add(url);
+            if (_isPotentialShortLink(url, allowedHosts)) unique.add(url);
           });
         });
       });
@@ -6669,9 +6927,9 @@ var require_index = __commonJS({
       }
       return null;
     }
-    function _applySpec(record, spec, cache) {
+    function _applySpec(record, spec, cache, resolvable) {
       const value = record[spec.field];
-      const resolve = (url) => typeof url === "string" && cache.has(url) ? cache.get(url) : url;
+      const resolve = (url) => typeof url === "string" && resolvable.has(url) && cache.has(url) ? cache.get(url) : url;
       if (spec.urlKey) {
         const original = value && value[spec.urlKey];
         const resolved = resolve(original);
@@ -6696,7 +6954,15 @@ var require_index = __commonJS({
     }
     function parseShortLinkDomains(value) {
       if (!value) return [];
-      return String(value).split(/[,;\s]+/).map((entry) => entry.replace(/^[a-z]+:\/\//i, "").split("/")[0].split(":")[0].replace(/\.$/, "").trim().toLowerCase()).filter((host) => host.includes("."));
+      return String(value).split(/[,;\s]+/).map((entry) => entry.replace(/^[a-z]+:\/\//i, "").replace(/^\*?\./, "").split("/")[0].split(":")[0].replace(/\.$/, "").trim().toLowerCase()).map(_toAsciiHost).filter((host) => host.includes(".") && !/^\d+(\.\d+){3}$/.test(host));
+    }
+    function _toAsciiHost(host) {
+      if (!host) return "";
+      try {
+        return new URL(`https://${host}`).hostname;
+      } catch (_error) {
+        return "";
+      }
     }
     const SHORT_LINKS_STATE_TTL_MS = 30 * 24 * 60 * 60 * 1e3;
     const SHORT_LINKS_STATE_MAX_BYTES = 24 * 1024;
@@ -6725,60 +6991,94 @@ var require_index = __commonJS({
       return state;
     }
     const UTM_PARAM_PATTERN = /utm_(source|medium|campaign|term|content)/;
-    function _isPotentialShortLink(url, nestedPathHosts) {
+    const KNOWN_SHORT_LINK_DOMAINS = [
+      "bit.ly",
+      "tinyurl.com",
+      "t.co",
+      "lnkd.in",
+      "youtu.be",
+      "amzn.to",
+      "ow.ly",
+      "buff.ly",
+      "cutt.ly",
+      "is.gd",
+      "rebrand.ly"
+    ];
+    function _isPotentialShortLink(url, allowedHosts) {
       if (!url || typeof url !== "string") return false;
       try {
         const parsedUrl = new URL(url);
         if (parsedUrl.protocol !== "https:" || url.includes("?") || UTM_PARAM_PATTERN.test(url)) {
           return false;
         }
-        const isConfiguredHost = _isNestedPathHost(parsedUrl.hostname, nestedPathHosts);
-        const pathSegments = _getPathSegments(parsedUrl.pathname, isConfiguredHost);
-        if (!pathSegments.every(Boolean)) return false;
-        return pathSegments.length === 1 || isConfiguredHost;
+        if (!_isAllowedShortLinkHost(parsedUrl.hostname, allowedHosts)) return false;
+        const pathSegments = _getPathSegments(parsedUrl.pathname);
+        return pathSegments.every(Boolean);
       } catch (_error) {
         return false;
       }
     }
-    function _getPathSegments(pathname, allowTrailingSlash) {
+    function _getPathSegments(pathname) {
       const segments = pathname.slice(1).split("/");
       const hasTrailingSlash = segments.length > 1 && segments[segments.length - 1] === "";
-      return allowTrailingSlash && hasTrailingSlash ? segments.slice(0, -1) : segments;
+      return hasTrailingSlash ? segments.slice(0, -1) : segments;
     }
-    function _isNestedPathHost(hostname, nestedPathHosts) {
-      return nestedPathHosts.some((host) => hostname === host || hostname.endsWith(`.${host}`));
+    function _isAllowedShortLinkHost(hostname, allowedHosts) {
+      const isMatch = (host) => hostname === host || hostname.endsWith(`.${host}`);
+      return KNOWN_SHORT_LINK_DOMAINS.some(isMatch) || allowedHosts.some(isMatch);
     }
     const SHORT_LINK_FETCH_TIMEOUT_MS = 1e4;
     const SHORT_LINK_MAX_REDIRECTS = 20;
     const SHORT_LINK_CONCURRENCY = 10;
-    async function _resolveShortLinks(shortLinks) {
+    async function _resolveShortLinks(shortLinks, allowedHosts = []) {
       const results = [];
       for (let i = 0; i < shortLinks.length; i += SHORT_LINK_CONCURRENCY) {
         const batch = shortLinks.slice(i, i + SHORT_LINK_CONCURRENCY);
-        results.push(...await Promise.all(batch.map(_resolveShortLink)));
+        results.push(...await Promise.all(batch.map((link) => _resolveShortLink(link, allowedHosts))));
       }
       return results;
     }
-    async function _resolveShortLink(linkObj) {
+    async function _resolveShortLink(linkObj, allowedHosts = []) {
       const originalUrl = linkObj.originalUrl;
       try {
-        return { originalUrl, resolvedUrl: await _followRedirects(originalUrl), ok: true };
+        return { originalUrl, resolvedUrl: await _followRedirects(originalUrl, allowedHosts), ok: true };
       } catch (error) {
-        console.log(`Failed to resolve short link ${originalUrl}: ${error.message}`);
-        return { originalUrl, resolvedUrl: originalUrl, ok: false };
+        console.log(`Failed to resolve short link ${_describeUrl(originalUrl)}: ${error.message}`);
+        return { originalUrl, resolvedUrl: originalUrl, ok: Boolean(error.isRefusal) };
+      }
+    }
+    function _describeUrl(url) {
+      try {
+        const { origin, pathname } = new URL(url);
+        return `${origin}${pathname}`;
+      } catch (_error) {
+        return "[unparseable URL]";
       }
     }
-    async function _followRedirects(startUrl) {
+    const SHORT_LINK_TRANSIENT_STATUS = (status) => status === 408 || status === 429 || status >= 500;
+    async function _followRedirects(startUrl, allowedHosts = []) {
+      var _a;
       let currentUrl = startUrl;
       for (let hop = 0; hop <= SHORT_LINK_MAX_REDIRECTS; hop++) {
         if (!_isPublicHttpUrl(currentUrl)) {
-          throw new Error(`Refusing to request non-public URL ${currentUrl}`);
+          const refusal2 = new Error(`Refusing non-public URL ${_describeUrl(currentUrl)}`);
+          refusal2.isRefusal = true;
+          throw refusal2;
         }
+        if (hop > 0 && !_isAllowedShortLinkHost(new URL(currentUrl).hostname, allowedHosts)) {
+          return currentUrl;
+        }
+        await new SsrfGuard([]).assertPublicHttps(currentUrl);
         const response = await fetch(currentUrl, {
           method: "GET",
           redirect: "manual",
           signal: AbortSignal.timeout(SHORT_LINK_FETCH_TIMEOUT_MS)
         });
+        await ((_a = response.body) == null ? void 0 : _a.cancel().catch(() => {
+        }));
+        if (SHORT_LINK_TRANSIENT_STATUS(response.status)) {
+          throw new Error(`Short link service answered ${response.status}`);
+        }
         const location = _getRedirectLocation(response);
         if (!location) return currentUrl;
         currentUrl = new URL(location, currentUrl).toString();
@@ -7087,7 +7387,7 @@ var require_index = __commonJS({
             let columnName = tableColumns[i];
             let columnDescription = "";
             if (!(columnName in this.schema)) {
-              throw new Error(`Required field ${columnName} not found in schema`);
+              throw this.missingSelectedFieldError(columnName);
             }
             let columnType = this.getColumnType(columnName);
             if ("description" in this.schema[columnName]) {
@@ -7715,7 +8015,7 @@ ${columnDef}`);
             let columnName = tableColumns[i];
             let columnDescription = "";
             if (!(columnName in this.schema)) {
-              throw new Error(`Required field ${columnName} not found in schema`);
+              throw this.missingSelectedFieldError(columnName);
             }
             let columnType = this.getColumnType(columnName);
             if ("description" in this.schema[columnName]) {
@@ -8741,7 +9041,7 @@ COMMIT TRANSACTION;`;
             let columnName = tableColumns[i];
             let columnDescription = "";
             if (!(columnName in this.schema)) {
-              throw new Error(`Required field ${columnName} not found in schema`);
+              throw this.missingSelectedFieldError(columnName);
             }
             let columnType = this.getColumnType(columnName);
             const isPartOfPK = pkColumns.includes(columnName);
@@ -12344,7 +12644,7 @@ Query: ${query}`);
               requiredType: "boolean",
               default: true,
               label: "Process Short Links",
-              description: "Resolve short links in landing URL fields to their landing page. The resolved value is written to the matching _parsed field (for example website_url_parsed)",
+              description: "Resolve short links in landing URL fields to the address their short link service points to. The resolved value is written to the matching _parsed field (for example website_url_parsed)",
               attributes: [CONFIG_ATTRIBUTES2.ADVANCED]
             },
             CreateEmptyTables: {
@@ -14336,7 +14636,7 @@ Query: ${query}`);
               requiredType: "boolean",
               default: true,
               label: "Process Short Links",
-              description: "Resolve short links in landing URL fields to their landing page. The resolved value is written to the matching _parsed field (for example landing_page_url_parsed)",
+              description: "Resolve short links in landing URL fields to the address their short link service points to. The resolved value is written to the matching _parsed field (for example landing_page_url_parsed)",
               attributes: [CONFIG_ATTRIBUTES2.ADVANCED]
             },
             CreateEmptyTables: {
@@ -19970,7 +20270,7 @@ Query: ${query}`);
               requiredType: "boolean",
               default: true,
               label: "Process Short Links",
-              description: "Resolve short links in landing URL fields to their landing page. The resolved value is written to the matching _parsed field (for example click_url_parsed)",
+              description: "Resolve short links in landing URL fields to the address their short link service points to. The resolved value is written to the matching _parsed field (for example click_url_parsed)",
               attributes: [CONFIG_ATTRIBUTES2.ADVANCED]
             },
             CreateEmptyTables: {
@@ -22147,7 +22447,7 @@ API Response: ${JSON.stringify(statusResult, null, 2)}`);
               requiredType: "boolean",
               default: true,
               label: "Process Short Links",
-              description: "Resolve short links in landing URL fields to their landing page. The resolved value is written to the matching _parsed field (for example FinalUrlParsed)",
+              description: "Resolve short links in landing URL fields to the address their short link service points to. The resolved value is written to the matching _parsed field (for example FinalUrlParsed)",
               attributes: [CONFIG_ATTRIBUTES2.ADVANCED]
             },
             CreateEmptyTables: {
@@ -26850,7 +27150,7 @@ API Response: ${JSON.stringify(statusResult, null, 2)}`);
               requiredType: "boolean",
               default: true,
               label: "Process Short Links",
-              description: "Resolve short links in landing URL fields to their landing page. The resolved value is written to the matching _parsed field (for example ad_final_urls_parsed)",
+              description: "Resolve short links in landing URL fields to the address their short link service points to. The resolved value is written to the matching _parsed field (for example ad_final_urls_parsed)",
               attributes: [CONFIG_ATTRIBUTES2.ADVANCED]
             },
             CreateEmptyTables: {
@@ -34399,7 +34699,7 @@ API Response: ${JSON.stringify(statusResult, null, 2)}`);
               requiredType: "boolean",
               default: true,
               label: "Process Short Links",
-              description: "Resolve short links to their landing page: link_url_asset.parsed_url on the Ad Insights by Link URL Asset endpoint, and link_url_parsed / object_url_parsed on Ad Creatives",
+              description: "Resolve short links to the address their short link service points to: link_url_asset.parsed_url on the Ad Insights by Link URL Asset endpoint, and link_url_parsed / object_url_parsed on Ad Creatives",
               attributes: [CONFIG_ATTRIBUTES2.ADVANCED]
             },
             CreateEmptyTables: {