@syncmatters/connector-sdk 1.0.0 → 1.0.1

package/docs/12-example-connector.md

@@ -0,0 +1,566 @@
+# Acme — a complete worked connector
+
+One canonical, end-to-end connector on a single page. Three objects — `companies`, `contacts`
+(first/last name matching, `upsertClean`) and `salesorders` (a **composite key**:
+business unit + order id) — plus a contact→company relationship, static + API-discovered
+custom fields, and full `query` / `upsert` / `delete`. Every pattern here follows the
+production fleet; copy this shape and adapt it. Topic-file cross-references are in comments.
+
+The imaginary Acme API: bearer-token REST at `https://api.example.com/v1/`, cursor pagination
+(`?after=<id>`), `modified_since` differential filter, a `/search` endpoint taking field
+criteria, batch write endpoints for companies/contacts, per-unit sales-order routes
+(`salesorders/{businessUnit}/{orderId}` — order ids are only unique within a business unit),
+and a `/fields` endpoint listing account-specific custom fields.
+
+## `files/Connectors/Acme/Acme.mjs`
+
+```js
+import SDK from "@syncmatters/connector-sdk";
+
+/**
+ * Acme connector - canonical example.
+ * Authoring guide: node_modules/@syncmatters/connector-sdk/docs/connector-authoring.md
+ */
+export default class Acme {
+  /** @type {SDK.Logger} */ #log;
+  /** @type {SDK.HttpClient} */ #http;
+  /** @type {string} */ #apiKey = "";
+  /** @type {boolean} */ #debug = false;
+  #baseUrl = "https://api.example.com/v1/";
+  // checkpoint overlap: rows modified while a query runs must be re-read next run (05, 10)
+  #checkpointOverlapMs = 5 * 60 * 1000;
+
+  /** @param {SDK.InitArgs} args @returns {Promise<void>} */
+  async init(args) {
+    SDK.verifyType(this);
+    this.#log = args.log;
+    this.#debug = args.settings.get("debug") === true;
+
+    this.#apiKey = args.settings.get("api_key");
+    if (!this.#apiKey) {
+      throw new SDK.ConnectorError(SDK.ErrorCode.InvalidOptionValue, "Setting 'API Key' is required");
+    }
+
+    // all HTTP through one rate-limited client; retry reads and throttle/gateway blips (03)
+    this.#http = SDK.utilities.httpClient({
+      limiter: SDK.utilities.rateLimiter({ type: "rolling", count: 10, ms: 1000 }),
+      maxRetry: 3,
+      retryBackoffMs: 2000,
+      canRetry: (err, method, url, httpCode) => {
+        if ([400, 401, 403, 404].includes(httpCode)) return false;
+        if (method === "GET") return true;
+        return [429, 502, 504].includes(httpCode);
+      },
+      defaultHeaders: { "Content-Type": "application/json" },
+    });
+  }
+
+  /** Central request wrapper: base URL + auth + debug logging (03). @param {SDK.HttpRequest} request */
+  async #call(request) {
+    if (!request.url.startsWith("https://")) request.url = this.#baseUrl + request.url;
+    request.headers = { ...request.headers, Authorization: `Bearer ${this.#apiKey}` };
+    try {
+      const response = await this.#http.execute(request);
+      this.#debug && this.#log.info(`DEBUG: ${request.method} ${request.url}`, { response });
+      return response;
+    } catch (err) {
+      this.#debug && this.#log.info(`DEBUG: ${request.method} ${request.url} failed`, { err });
+      throw err;
+    }
+  }
+
+  /** Cheapest authenticated call; catch, never throw (02). @returns {Promise<SDK.TestResult>} */
+  async test() {
+    try {
+      await this.#call({ method: "GET", url: "me" });
+      return { success: true };
+    } catch (err) {
+      return { success: false, message: err instanceof Error ? err.message : String(err) };
+    }
+  }
+
+  // ---------------------------------------------------------------- meta (04)
+
+  /** @returns {Promise<SDK.ConnectorMeta>} */
+  async meta() {
+    const me = await this.#call({ method: "GET", url: "me" });
+    const [companyCustom, contactCustom] = await Promise.all([
+      this.#customFields("companies"),
+      this.#customFields("contacts"),
+    ]);
+
+    /** @type {SDK.ObjectMeta[]} */
+    const objects = [
+      {
+        id: "companies",
+        name: "Companies",
+        canQueryList: true,
+        canQueryByIds: true,
+        canQueryByCheckpoint: true,
+        matchRules: ["id", "name[ci]"],
+        queryFields: [
+          { id: "id", name: "ID", type: "string", isKey: true, canMatch: true },
+          { id: "name", name: "Name", type: "string", canMatch: true },
+          { id: "domain", name: "Domain", type: "string", canMatch: true },
+          { id: "created_at", name: "Created", type: "string" },
+          ...companyCustom,
+        ],
+        // upsert accepts the writable statics + customs; the key addresses updates (04)
+        upsertFields: [
+          { id: "name", name: "Name", type: "string", constraints: { mandatory: "add" } },
+          { id: "domain", name: "Domain", type: "string" },
+          ...companyCustom,
+          { id: "id", type: "string", isKey: true },
+        ],
+        deleteFields: [{ id: "id", type: "string", isKey: true }],
+      },
+      {
+        id: "contacts",
+        name: "Contacts",
+        canQueryList: true,
+        canQueryByIds: true,
+        canQueryByCheckpoint: true,
+        canUpsertClean: true, // we implement upsertClean() below (06)
+        matchRules: ["id", "email[ci]", "first_and_last_name[ci]"],
+        relationships: [
+          // enables relatedFilter queries; data.idField is OUR payload, echoed back in
+          // options.meta so #relatedWhere below can find the FK (04, 05)
+          { id: "companies-company_id", name: "Company", relObjectId: "companies",
+            cardinality: "OneToOne", data: { idField: "company_id" } },
+        ],
+        queryFields: [
+          { id: "id", name: "ID", type: "string", isKey: true, canMatch: true },
+          { id: "email", name: "Email", type: "string", canMatch: true, constraints: { email: true } },
+          { id: "first_name", name: "First Name", type: "string", canMatch: true },
+          { id: "last_name", name: "Last Name", type: "string", canMatch: true },
+          { id: "company_id", name: "Company ID", type: "string" },
+          ...contactCustom,
+        ],
+        upsertFields: [
+          { id: "email", name: "Email", type: "string", constraints: { mandatory: "add", email: true } },
+          { id: "first_name", name: "First Name", type: "string" },
+          { id: "last_name", name: "Last Name", type: "string" },
+          { id: "company_id", name: "Company ID", type: "string" },
+          ...contactCustom,
+          { id: "id", type: "string", isKey: true },
+        ],
+        deleteFields: [{ id: "id", type: "string", isKey: true }],
+      },
+      {
+        // COMPOSITE KEY object: order_id is only unique within a business_unit, so the row
+        // identity is BOTH - encoded into the $compositeKey pseudo-field (11)
+        id: "salesorders",
+        name: "Sales Orders",
+        canQueryList: true,
+        canQueryByIds: true,
+        queryFields: [
+          { id: "$compositeKey", name: "Composite key", type: "string", isKey: true },
+          { id: "business_unit", name: "Business Unit", type: "string" },
+          { id: "order_id", name: "Order ID", type: "number" },
+          { id: "amount", name: "Amount", type: "number" },
+          { id: "status", name: "Status", type: "string" },
+        ],
+        upsertFields: [
+          { id: "business_unit", name: "Business Unit", type: "string", constraints: { mandatory: "add" } },
+          { id: "amount", name: "Amount", type: "number" },
+          { id: "status", name: "Status", type: "string" },
+          { id: "$compositeKey", name: "Composite key", type: "string", isKey: true },
+        ],
+        deleteFields: [{ id: "$compositeKey", name: "Composite key", type: "string", isKey: true }],
+      },
+    ];
+
+    // stable account identity - platform detects re-pointed connections with it (04)
+    return { identity: `${me.account_name}|${me.email}`, objects };
+  }
+
+  /** Discover account-specific custom fields via the API. @param {string} objectId @returns {Promise<SDK.ObjectField[]>} */
+  async #customFields(objectId) {
+    const resp = await this.#call({ method: "GET", url: `${objectId}/fields` });
+    /** @type {SDK.ObjectField[]} */
+    const fields = [];
+    for (const f of resp.fields) {
+      /** @type {SDK.ObjectField} */
+      const field = { id: f.key, name: f.label, type: "string", isCustom: true };
+      switch (f.type) {
+        case "number":  field.type = "number"; break;
+        case "boolean": field.type = "boolean"; break;
+        case "picklist":
+          field.optionValues = f.options.map((/** @type {{id: string, label?: string}} */ o) =>
+            ({ id: o.id, name: o.label }));
+          break;
+        case "multipicklist":
+          field.type = "array";
+          field.arrayType = "string";
+          break;
+      }
+      fields.push(field);
+    }
+    return fields;
+  }
+
+  // ---------------------------------------------------------------- query (05)
+
+  /** @param {SDK.QueryOptions} options @returns {Promise<SDK.QueryPage>} */
+  async query(options) {
+    const qo = options.queryOptions || {};
+    // fail fast on filters this object never declared (08)
+    if (qo.checkpointFilter && !options.meta.canQueryByCheckpoint) {
+      throw new SDK.ConnectorError(SDK.ErrorCode.InvalidFilterParameter,
+        `object ${options.id} does not support checkpoint queries`);
+    }
+
+    if (qo.matchFilter) return await this.#queryMatch(options, qo.matchFilter);
+    // composite-key ids arrive as key STRINGS - parse each and fetch directly (11)
+    if (options.id === "salesorders" && qo.idsFilter) {
+      return await this.#querySalesOrdersByIds(qo.idsFilter);
+    }
+
+    const queryState = options.queryState ||
+      { after: undefined, limit: 200, rowCount: 0, startTime: Date.now() };
+
+    const params = [`limit=${queryState.limit}`];
+    if (queryState.after) params.push(`after=${encodeURIComponent(queryState.after)}`);
+
+    if (qo.idsFilter) {
+      const ids = qo.idsFilter.filter((id) => id != null && id !== ""); // never emit id=undefined (10)
+      if (ids.length === 0) return { rows: [], finalPage: true };
+      params.push(`ids=${ids.map(encodeURIComponent).join(",")}`);
+    }
+    if (qo.checkpointFilter) {
+      // undefined value = first differential run: return everything (05)
+      params.push(`modified_since=${encodeURIComponent(qo.checkpointFilter.value || "1970-01-01T00:00:00Z")}`);
+    }
+    if (qo.relatedFilter) params.push(this.#relatedWhere(options, qo.relatedFilter));
+
+    const resp = await this.#call({ method: "GET", url: `${options.id}?${params.join("&")}` });
+
+    /** @type {SDK.Row[]} */
+    let rows = resp.items.map((/** @type {any} */ item) => this.#row(options.id, item));
+    if (qo.relatedFilter) rows = this.#tagRelationships(rows, qo.relatedFilter);
+
+    queryState.after = resp.next_cursor || undefined;
+    queryState.rowCount += rows.length;
+
+    // respect the caller's limit: trim overflow, then stop (05)
+    if (qo.limit != null && queryState.rowCount > qo.limit) {
+      rows = rows.slice(0, rows.length - (queryState.rowCount - qo.limit));
+      queryState.rowCount = qo.limit;
+    }
+
+    /** @type {SDK.QueryPage} */
+    const result = { rows, queryState };
+    result.finalPage = rows.length === 0 || !queryState.after ||
+      (qo.limit != null && queryState.rowCount >= qo.limit);
+
+    if (qo.checkpointFilter && result.finalPage) {
+      // ALWAYS overlap - emit earlier than "now" so mid-run modifications are re-read (05, 10)
+      result.checkpoint = new Date(queryState.startTime - this.#checkpointOverlapMs).toISOString();
+    }
+    return result;
+  }
+
+  /** One place builds rows, so keys are ALWAYS encoded identically (11).
+   * @param {string} objectId @param {any} item @returns {SDK.Row} */
+  #row(objectId, item) {
+    if (objectId === "salesorders") {
+      const key = makeSalesOrderKey(item);
+      // expose the key as a field too, so callers can address/inspect it (11)
+      return { meta: { key }, data: { ...item, $compositeKey: key } };
+    }
+    return {
+      meta: { key: String(item.id), url: `https://app.example.com/${objectId}/${item.id}` },
+      data: item,
+    };
+  }
+
+  /** @param {string[]} keys @returns {Promise<SDK.QueryPage>} */
+  async #querySalesOrdersByIds(keys) {
+    const rows = await Promise.all(keys.filter((k) => k).map(async (key) => {
+      const { businessUnit, orderId } = parseSalesOrderKey(key);
+      const item = await this.#call({
+        method: "GET",
+        url: `salesorders/${encodeURIComponent(businessUnit)}/${orderId}`,
+      });
+      return this.#row("salesorders", item);
+    }));
+    return { rows, finalPage: true };
+  }
+
+  /** Match by email / first+last name / company name: search candidates, let the platform
+   * reserve winners via canUse (05).
+   * @param {SDK.QueryOptions} options @param {NonNullable<NonNullable<SDK.QueryOptions["queryOptions"]>["matchFilter"]>} matchFilter
+   * @returns {Promise<SDK.QueryPage>} */
+  async #queryMatch(options, matchFilter) {
+    if (!(options.meta.matchRules || []).includes(matchFilter.rule)) {
+      throw new SDK.ConnectorError(SDK.ErrorCode.InvalidFilterParameter,
+        `object ${options.id} does not support match rule '${matchFilter.rule}'`);
+    }
+    const candidates = [];
+    for (const src of matchFilter.srcData) {
+      const criteria = this.#matchCriteria(matchFilter.rule, src.match);
+      if (!criteria) continue; // src is missing a required match value - skip, never guess (10)
+      const resp = await this.#call({
+        method: "POST", url: `${options.id}/search`,
+        body: { criteria, case_insensitive: true },
+      });
+      if (resp.items.length > 0) {
+        candidates.push({
+          srcRowId: src.srcRowId,
+          candidates: resp.items.map((/** @type {any} */ item) => this.#row(options.id, item)),
+        });
+      }
+    }
+    // canUse reserves un-mapped matches and returns rows with row.relationship populated (05)
+    const rows = await matchFilter.canUse(candidates);
+    return { rows, finalPage: true };
+  }
+
+  /** Map a match rule to API search criteria; undefined = this src can't be searched.
+   * @param {string} rule @param {{id?: string, name?: string, email?: string, firstname?: string, lastname?: string}} match */
+  #matchCriteria(rule, match) {
+    switch (rule) {
+      case "email[ci]":
+        return match.email ? { email: match.email } : undefined;
+      case "name[ci]":
+        return match.name ? { name: match.name } : undefined;
+      case "first_and_last_name[ci]":
+        // BOTH parts are required - a first name alone must not match (05)
+        return match.firstname && match.lastname
+          ? { first_name: match.firstname, last_name: match.lastname }
+          : undefined;
+      default:
+        return undefined;
+    }
+  }
+
+  /** Resolve the FK declared in meta().relationships via its data payload.
+   * @param {SDK.QueryOptions} options @param {SDK.QueryRelatedFilter} related */
+  #relatedWhere(options, related) {
+    const rel = (options.meta.relationships || []).find((r) => r.id === related.otherRelationshipId);
+    if (!rel) {
+      throw new SDK.ConnectorError(SDK.ErrorCode.InvalidFilterParameter,
+        `object ${options.id} has no relationship '${related.otherRelationshipId}'`);
+    }
+    const fk = /** @type {{idField: string}} */ (rel.data).idField;
+    const ids = related.otherRows.map((r) => r.meta.key);
+    return `${fk}=${ids.map(encodeURIComponent).join(",")}`;
+  }
+
+  /** One row instance per source row it relates to, tagged with the relationship (05).
+   * @param {SDK.Row[]} rows @param {SDK.QueryRelatedFilter} related @returns {SDK.Row[]} */
+  #tagRelationships(rows, related) {
+    const out = [];
+    for (const row of rows) {
+      for (const src of related.otherRows) {
+        if (String(row.data.company_id) === src.meta.key) {
+          out.push({ ...row, relationship: {
+            srcRowId: src.meta.key,
+            srcObjectId: related.otherObjectId,
+            relationshipId: related.otherRelationshipId,
+          } });
+        }
+      }
+    }
+    return out;
+  }
+
+  // ---------------------------------------------------------------- upsert / delete (06)
+
+  /** Pre-write validation + change detection - called for contacts (canUpsertClean).
+   * ORDER MATTERS: the verifies can MUTATE the upsert (cleanse invalid picklist options,
+   * resolve labels to ids), so detect the change LAST (06).
+   * @param {SDK.UpsertCleanOptions} options @returns {Promise<SDK.UpsertCleanResponse>} */
+  async upsertClean(options) {
+    const issues = [
+      ...(await options.verifyFieldOptions({ cleanAction: "DeleteUpsertField" })),
+      ...(await options.verifyFieldConstraints({ timeZone: "UTC" })),
+    ];
+    const change = options.detectFirstChange();
+    return { hasChanges: change !== undefined, change, issues };
+  }
+
+  /** @param {SDK.UpsertOptions} options @returns {Promise<SDK.Row[]>} */
+  async upsert(options) {
+    if (options.id === "salesorders") return await this.#upsertSalesOrders(options);
+
+    // split add vs update, batch each, then re-align POSITIONALLY with options.rows (06, 10)
+    /** @type {any[]} */ const inserts = [];
+    /** @type {any[]} */ const updates = [];
+    /** @type {Map<number, {list: "i"|"u", at: number}>} */ const where = new Map();
+    options.rows.forEach((row, i) => {
+      if (row.id != null) { updates.push(row); where.set(i, { list: "u", at: updates.length - 1 }); }
+      else { inserts.push(row); where.set(i, { list: "i", at: inserts.length - 1 }); }
+    });
+
+    const iOut = await this.#writeBatches(options.id, "POST", "batch/create", inserts);
+    const uOut = await this.#writeBatches(options.id, "POST", "batch/update", updates);
+
+    return options.rows.map((_, i) => {
+      const w = /** @type {{list: "i"|"u", at: number}} */ (where.get(i));
+      const row = w.list === "i" ? iOut[w.at] : uOut[w.at];
+      if (!row) {
+        // never guess at correlation - misalignment corrupts data silently (10)
+        throw new SDK.ConnectorError(SDK.ErrorCode.InvalidOperation,
+          `upsert result for input row ${i} could not be correlated`);
+      }
+      return row;
+    });
+  }
+
+  /** Composite-key writes: parse the key for updates, require the parent part for adds (11).
+   * Promise.all over map keeps results positionally aligned with options.rows.
+   * @param {SDK.UpsertOptions} options @returns {Promise<SDK.Row[]>} */
+  async #upsertSalesOrders(options) {
+    return await Promise.all(options.rows.map(async (/** @type {any} */ row) => {
+      const compositeKey = row["$compositeKey"];
+      if (compositeKey) {
+        // UPDATE: decode the key for the URL; the pseudo-field never goes to the API (11)
+        const { businessUnit, orderId } = parseSalesOrderKey(compositeKey);
+        const body = { ...row };
+        delete body["$compositeKey"];
+        const result = await this.#call({
+          method: "PUT",
+          url: `salesorders/${encodeURIComponent(businessUnit)}/${orderId}`,
+          body,
+        });
+        // echo the EXACT key string we received - never re-encode (11)
+        return { meta: { key: compositeKey }, data: { ...result, $compositeKey: compositeKey } };
+      }
+      // ADD: no key yet - the parent part must come as a regular field
+      if (row.business_unit == null) {
+        throw new SDK.ConnectorError(SDK.ErrorCode.InvalidApiKey,
+          "adding a salesorder requires 'business_unit'");
+      }
+      const result = await this.#call({
+        method: "POST",
+        url: `salesorders/${encodeURIComponent(row.business_unit)}`,
+        body: row,
+      });
+      return this.#row("salesorders", result);
+    }));
+  }
+
+  /** Batch writes in API-sized chunks; per-row API failures throw ApiReportedError (06, 08).
+   * @param {string} objectId @param {"POST"} method @param {string} action @param {any[]} rows
+   * @returns {Promise<SDK.Row[]>} */
+  async #writeBatches(objectId, method, action, rows) {
+    const batchSize = 100;
+    /** @type {SDK.Row[]} */ const out = [];
+    for (let i = 0; i < rows.length; i += batchSize) {
+      const batch = rows.slice(i, i + batchSize);
+      const resp = await this.#call({ method, url: `${objectId}/${action}`, body: { inputs: batch } });
+      for (const result of resp.results) { // API echoes results in input order
+        if (result.error) {
+          this.#log.error(`Acme ${action} reported an error`, result);
+          throw new SDK.ConnectorError(SDK.ErrorCode.ApiReportedError,
+            `Acme ${action} reported an error: ${JSON.stringify(result.error)}`);
+        }
+        out.push(this.#row(objectId, result));
+      }
+    }
+    return out;
+  }
+
+  /** @param {SDK.DeleteOptions} options @returns {Promise<SDK.Row[]>} */
+  async delete(options) {
+    if (options.id === "salesorders") {
+      return await Promise.all(options.rows.map(async (/** @type {any} */ row) => {
+        const key = row["$compositeKey"];
+        const { businessUnit, orderId } = parseSalesOrderKey(key);
+        await this.#call({
+          method: "DELETE",
+          url: `salesorders/${encodeURIComponent(businessUnit)}/${orderId}`,
+        });
+        // echo the exact key back so the platform matches the cached row (11)
+        return { meta: { key, deleted: true }, data: { business_unit: businessUnit, order_id: orderId } };
+      }));
+    }
+    const ids = options.rows.map((r) => String(r.id));
+    await this.#call({ method: "POST", url: `${options.id}/batch/archive`, body: { ids } });
+    return options.rows.map((r) => ({ meta: { key: String(r.id), deleted: true }, data: { id: r.id } }));
+  }
+}
+
+// ------------------------------------------------------------- composite key helpers (11)
+// salesorders identity = (business_unit, order_id). Encoded as ONE deterministic JSON string
+// with sorted parts - the platform compares keys by string equality, so the same row must
+// produce a byte-identical key on every query, forever. Never reorder or re-format.
+
+/** @param {any} data @returns {string} */
+function makeSalesOrderKey(data) {
+  if (data.business_unit == null || data.order_id == null) {
+    throw new SDK.ConnectorError(SDK.ErrorCode.InvalidApiKey,
+      `cannot build salesorders key: business_unit/order_id missing: ${JSON.stringify(data)}`);
+  }
+  // property order fixed by construction (sorted): business_unit, then order_id
+  return JSON.stringify({ business_unit: data.business_unit, order_id: data.order_id });
+}
+
+/** @param {string} compositeKey @returns {{businessUnit: string, orderId: number}} */
+function parseSalesOrderKey(compositeKey) {
+  /** @type {any} */
+  let parsed;
+  try {
+    parsed = JSON.parse(compositeKey);
+  } catch {
+    throw new SDK.ConnectorError(SDK.ErrorCode.InvalidApiKey,
+      `failed to parse salesorders composite key '${compositeKey}'`);
+  }
+  if (parsed?.business_unit == null || parsed?.order_id == null) {
+    throw new SDK.ConnectorError(SDK.ErrorCode.InvalidApiKey,
+      `salesorders composite key '${compositeKey}' did not contain business_unit + order_id`);
+  }
+  return { businessUnit: String(parsed.business_unit), orderId: Number(parsed.order_id) };
+}
+```
+
+## The sidecar: `meta/Connectors/Acme/Acme.mjs.meta.json`
+
+```json
+{
+  "runnable": false,
+  "type": "connector",
+  "cpu_architecture": "",
+  "engine": 0,
+  "resource_size": "medium",
+  "name": "Acme.mjs",
+  "module_script_file_paths": [],
+  "connector_metadata": {
+    "categories": ["sales"],
+    "agent_mode": "never",
+    "settings": {
+      "api_key": { "order": "00", "name": "API Key", "type": "string", "control": "singlelinetext", "secret": true },
+      "debug":   { "order": "01", "name": "Debug", "advanced": true, "type": "boolean", "control": "singlelinetext" }
+    }
+  }
+}
+```
+
+## What to notice (the parts assistants most often get wrong)
+
+1. **One `#call` wrapper** owns base URL, auth and debug logging; nothing else touches HTTP.
+2. **`meta()` is generated, not hand-typed twice** — static fields inline, custom fields
+   discovered from the API and spread into BOTH `queryFields` and `upsertFields`; the `isKey`
+   field is appended to `upsertFields` so updates can address rows.
+3. **One `#row()` builds every row**, so keys are always encoded identically — including the
+   composite `salesorders` key, which is built by a single `makeSalesOrderKey` and **echoed
+   verbatim** (never re-encoded) on update/delete.
+4. **The composite `$compositeKey` pseudo-field carries `isKey`** and is stripped from every
+   API body; adds supply the parent part (`business_unit`) as a regular mandatory field.
+5. **`relationships[].data` is the connector's own round-trip payload** — `meta()` stores the
+   FK field name, `query()` reads it back from `options.meta` when resolving `relatedFilter`.
+6. **`query()` dispatches filters in one place**, throws `InvalidFilterParameter` for anything
+   undeclared, and shares the pagination path across list/ids/checkpoint variants — except
+   composite-key `idsFilter`, which parses each key and fetches directly.
+7. **The checkpoint is `startTime - overlap`**, emitted only on the final page.
+8. **Match queries end with `canUse`** — the platform picks the winners, you just supply
+   candidates. `first_and_last_name[ci]` requires BOTH parts; sources missing either are
+   skipped, never half-matched.
+9. **`upsertClean` runs the mutating verifies first, `detectFirstChange` last**, and only the
+   object that declares `canUpsertClean` (contacts) is cleaned.
+10. **Upsert output is index-mapped back to input order** and throws rather than guessing when
+    a result can't be correlated (`Promise.all` over `map` preserves order for the per-row
+    sales-order path).
+11. Every method is JSDoc-typed against `SDK.*`; state lives in `#fields`; the only throw
+    types are `ConnectorError` (and propagated `HttpError`s).

package/docs/13-file-fields.md

@@ -0,0 +1,123 @@
+# File fields
+
+Objects can carry file content (documents, attachments, exports) as **fields of
+`type: "file"`** — declared like any other field, requested like any other field, and written
+like any other field. The value of a file field is an `SDK.FileProvider`.
+
+## Declaring
+
+```js
+// meta() - a document object whose content is a file field
+{
+  id: "documents",
+  name: "Documents",
+  canQueryList: true,
+  canQueryByIds: true,
+  queryFields: [
+    { id: "id", name: "ID", type: "string", isKey: true },
+    { id: "name", name: "File name", type: "string" },
+    { id: "size", name: "Size (bytes)", type: "number" },
+    { id: "content", name: "Content", type: "file" },
+  ],
+  upsertFields: [
+    { id: "name", name: "File name", type: "string", constraints: { mandatory: "add" } },
+    { id: "content", name: "Content", type: "file" },
+    { id: "id", type: "string", isKey: true },
+  ],
+}
+```
+
+## Querying: include the file only when asked
+
+File content is expensive — never fetch it unconditionally. The caller signals what it wants
+through `options.queryOptions.fields` (the field paths it expects, see
+[05-query.md](./05-query.md)); include file content **only when a file-typed field is in that
+list**:
+
+```js
+/** does the caller want any of this object's file fields? */
+#wantsField(options, fieldId) {
+  const fields = options.queryOptions?.fields;
+  if (!fields) return false; // no field selection -> cheap fields only, no file content
+  return fields.some((path) => path[0]?.path === fieldId);
+}
+
+async query(options) {
+  const wantsContent = this.#wantsField(options, "content");
+  // ...fetch the row list as usual...
+  const rows = [];
+  for (const item of resp.items) {
+    /** @type {any} */
+    const data = { id: item.id, name: item.name, size: item.size };
+    if (wantsContent) {
+      // fileResponse: true -> the http client hands back an SDK.FileProvider
+      data.content = /** @type {SDK.FileProvider} */ (await this.#http.execute({
+        method: "GET",
+        url: `documents/${item.id}/content`,
+        fileResponse: true,
+      }));
+    }
+    rows.push({ meta: { key: String(item.id) }, data });
+  }
+  // ...
+}
+```
+
+`fileResponse: true` on an `HttpRequest` is the standard way to download into a provider
+without buffering the whole body in memory. To build a provider from something you already
+have, use the constructor variants:
+
+```js
+SDK.utilities.fileProvider({ buffer: someBuffer });
+SDK.utilities.fileProvider({ string: someText });
+SDK.utilities.fileProvider({ file: { path: tempPath, deleteOnClose: true } });
+```
+
+## Consuming a `FileProvider`
+
+| Method                                | Use                                                                     |
+| ------------------------------------- | ----------------------------------------------------------------------- |
+| `stream(encoding?)`                   | a readable stream — the normal way to upload or parse without buffering |
+| `blob(fileName?)` / `formDataValue()` | for APIs consumed via `fetch`-style multipart bodies                    |
+| `save(path?)`                         | persist to disk, returns the path (pair with `SDK.utilities.tempFile`)  |
+| `length()`                            | size in bytes (e.g. for a `size` field or Content-Length header)        |
+| `close()`                             | release the resource — **always, in a `finally`**                       |
+
+Parsing structured file content: `SDK.utilities.csvReader({ source: { fileProvider } })`
+(same idea for `xlsxReader`).
+
+## Upserting: the file arrives as a field value
+
+The upsert row carries a `FileProvider` at the file field's position. Stream it out and close
+it:
+
+```js
+async upsert(options) {
+  return await Promise.all(options.rows.map(async (/** @type {any} */ row) => {
+    const file = /** @type {SDK.FileProvider} */ (row.content);
+    try {
+      const result = await this.#call({
+        method: "POST",
+        url: `documents${row.id ? `/${row.id}` : ""}`,
+        headers: { "Content-Type": "application/octet-stream", "X-File-Name": row.name },
+        body: await file.stream(),
+      });
+      return { meta: { key: String(result.id) }, data: result };
+    } finally {
+      try { await file.close(); } catch { /* already closed */ }
+    }
+  }));
+}
+```
+
+## Gotchas
+
+1. **No `fields` list means no file content.** When the caller doesn't select fields, return
+   the cheap scalar fields only — a list query that downloads every document is a platform
+   incident waiting to happen.
+2. **`upsertClean` treats every `FileProvider` as a change** and `confirmChange` cannot veto
+   it ([06-upsert-delete.md](./06-upsert-delete.md)) — keep file fields out of cleaned
+   payloads or accept the always-write.
+3. **Close what you open.** Providers can be backed by temp files; `close()` in a `finally`
+   on both the query and upsert paths.
+