@syncmatters/connector-sdk 1.0.5 → 1.0.7

package/docs/01-workspace-and-meta-json.md

@@ -69,7 +69,7 @@ await this.#core.init(args);
 | Field                      | Values / meaning                                                                                                                                                                                                                                         |
 | -------------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
 | `name`                     | the file name, e.g. `"Acme.mjs"`                                                                                                                                                                                                                         |
-| `type`                     | `"connector"` (entry) · `"module"` · `"connectortest"` · `"json"` · `"yaml"`                                                                                                                                                                             |
+| `type`                     | `"connector"` (entry) · `"module"` · `"connectortest"` · `"json"` · `"yaml"` · `"markdown"` (help docs shipped beside the connector, e.g. `Connectors/<Name>/help/index.md` — see [17-user-help-pages.md](./17-user-help-pages.md); never a module, never staged at runtime) |
 | `runnable`                 | `false` for connector files (they are invoked by the platform, not run directly)                                                                                                                                                                         |
 | `engine`                   | runtime engine version int; leave as scaffolded unless told otherwise                                                                                                                                                                                    |
 | `resource_size`            | `""` or `"medium"` — runtime resource class hint                                                                                                                                                                                                         |
@@ -105,16 +105,22 @@ Keyed by setting id; the values arrive in `init()` via `args.settings.get("<id>"
                 "type": "string", "control": "select",
                 "options": [ { "id": "1", "name": "1 (Superseded)", "order": "00" }, { "id": "2", "name": "2 (Recommended)", "order": "01" } ],
                 "default": { "type": "string", "expression": "2" } },
+  "time_zone":  { "order": "04", "name": "Time Zone", "type": "string", "control": "timezone",
+                "default": { "type": "string", "expression": "UTC" } },
   "debug":    { "order": "03", "name": "Debug", "advanced": true, "type": "boolean", "control": "singlelinetext" }
 }
 ```
 
 Per-setting fields: `order` (two-digit string, display order), `name`, `type` (`string` |
-`boolean`), `control` (`singlelinetext` | `select`), `secret: true` (masked + encrypted),
+`boolean`), `control` (`singlelinetext` | `textarea` | `select` | `timezone`), `secret: true` (masked + encrypted),
 `advanced: true` (collapsed by default), `hidden: true`, `readonly: true`, `placeholder`,
-`options` (for `select`), `default: { "type": "string", "expression": "<value>" }`, and
+`options` (for `select` only — not used for `timezone`), `default: { "type": "string", "expression": "<value>" }`, and
 `defines_version: true` on the setting whose value picks a version core.
 
+Use `control: "timezone"` for IANA time zone pickers (e.g. fleet `time_zone` setting). The stored
+value is a plain string (`"America/New_York"`, `"UTC"`, …); the UI renders a searchable list of
+IANA zones — do not list zones in `options`.
+
 Conventions worth copying: a secret `api_key`/`clientId`+`clientSecret` pair for credentials; an
 advanced boolean `debug` setting that gates verbose request/response logging; a
 `connectorVersion` select with `defines_version` when you need breaking revisions to coexist.

package/docs/04-meta-objects-fields.md

@@ -75,7 +75,7 @@ same as "no list support". See [the query capability bar](#the-query-capability-
 | `canUpdateOptions: true`          | new picklist options may be added via `upsertFieldOptions()`                                                                                                                      |
 | `objectFields`                    | child fields when `type: "object"`                                                                                                                                                |
 | `arrayType` / `arrayObjectFields` | element type / element fields when `type: "array"`                                                                                                                                |
-| `constraints`                     | validation: `mandatory` (`"always"` · `"add"` · `"update"`), `max/min/maxLen/minLen`, `regex`, `email`, `integer`, `decimals`, `trim`, `dt: { format: "iso8601", "unix", ... }`   |
+| `constraints`                     | validation: `mandatory` (`"always"` · `"add"` · `"update"`; array children only when parent array is non-empty), `max/min/maxLen/minLen`, `regex`, `email`, `integer`, `decimals`, `trim`, `dt: { format: "iso8601", "unix", ... }`   |
 | `isCustom`                        | account-specific/custom field                                                                                                                                                     |
 | `data`                            | connector-private payload, echoed back in metadata                                                                                                                                |
 
@@ -215,6 +215,11 @@ last-modified filter actually exists on that object.
 - `mandatory` (`"always" | "add" | "update"`) raises a `ValueRequired` issue — and
   `mandatoryAccepts: ["null" | "blank_string"]` lets a mandatory field treat `null`/`""` as a
   satisfying value (for APIs where "explicitly blank" is meaningful).
+- For fields inside **`arrayObjectFields`**, `mandatory` is only checked when the parent array
+  has at least one element. An empty array (`[]`), a missing array property, or a missing parent
+  object does **not** raise `ValueRequired` for child fields — e.g. `ContactLinks.list.[0-n].CompanyId`
+  with `mandatory: "always"` is required per link object, but not when `list` is empty.
+  When the array is non-empty, **every** element must satisfy the mandatory constraint.
 
 ## `identity`
 

package/docs/05-query.md

@@ -115,6 +115,40 @@ the final page; the platform stores it and hands it back as `checkpointFilter.va
   rows modified while your query ran would otherwise be lost forever. Re-reading a few rows is
   free; losing one is a data bug.
 
+#### Time zones
+
+**Emit the checkpoint as a UTC ISO-8601 string.** The value is opaque to the platform, but
+UTC keeps logs, the results UI and your own debugging sane — and wall-clock arithmetic across
+DST transitions is where checkpoint bugs breed.
+
+Some systems — especially on-prem APIs (SysPro is the fleet example) — expect their
+changed-since filter as **wall-clock time in the server's local zone**, not UTC. The pattern:
+
+1. Add a connection setting for the IANA time zone (fleet convention: `time_zone`, e.g.
+   `"America/New_York"`) with `type: "string"` and `control: "timezone"` — the connector cannot
+   guess the server's zone. See [01-workspace-and-meta-json.md](./01-workspace-and-meta-json.md).
+2. Keep the STORED checkpoint UTC; convert **at the filter**, when building the request from
+   `checkpointFilter.value`:
+
+```js
+const localSince = SDK.utilities.dateTransform({
+  input: options.queryOptions.checkpointFilter.value ?? "1970-01-01T00:00:00Z",
+  inputFormat: "iso8601",
+  outputFormat: "yyyy-MM-dd HH:mm:ss",   // or "custom" + outputFormatCustom for odd APIs
+  outputTimeZone: this.#timeZone,        // the connection's IANA time_zone setting
+});
+```
+
+`dateTransform` also carries `inputTimeZone` (parse a zone-less API timestamp as wall clock),
+`dateTimeMath` (the overlap subtraction: `[{ count: -5, interval: "minutes" }]`) and
+truncation; `dateFormat(date, { format, tz })` is the light variant for rendering a `Date`.
+
+DST note: converting through a wall-clock zone makes one hour a year ambiguous or absent —
+your overlap window is what absorbs it, which is one more reason the overlap rule is not
+optional. If you find fleet code emitting a wall-clock checkpoint with a re-tagged offset
+(SysPro does), that is a workaround for an API that round-trips its own local time — prefer
+UTC-stored/convert-at-filter for new connectors.
+
 ### rowFilter — user-parameterized augmentation of the standard filters
 
 The system-specific filters you declared in `meta().queryFilter.rowFilter`

package/docs/06-upsert-delete.md

@@ -108,7 +108,8 @@ async upsertClean(options) {
   auto-created via your `upsertFieldOptions()` (resolved ids written back into the upsert).
   **Because it mutates, always run the verifies BEFORE `detectFirstChange`** — the production
   fleet does, without exception.
-- `verifyFieldConstraints` enforces your `ObjectField.constraints`.
+- `verifyFieldConstraints` enforces your `ObjectField.constraints` (including `mandatory` on
+  array item fields — only when the parent array is non-empty; see [04](./04-meta-objects-fields.md)).
 - Issues are `{ type, fatal?, fieldPath?, value?, option? }`; `fatal: true` blocks the row.
 - Returning `hasChanges` with no fatal issues queues the row; the platform batches queued rows
   and calls your `upsert()` with them later (clean order = write order; `upsertClean` calls

package/docs/09-testing.md

@@ -112,14 +112,9 @@ To disable those tests use `cannotTestReason` instead
   (`rowFiltersPrepare` exists in the contract but the suite does NOT run rowFilter tests yet —
   when `meta()` exposes `queryFilter.rowFilter`, cover it yourself with a direct query in a
   hook you already implement.)
-- `checkpointFilterStep1Prepare` / `Step2Prepare` — step 1 returns `expectedRowIds` for a
-  query from your chosen starting checkpoint (`nextCheckpoint`); the harness runs it and
-  captures the checkpoint the query emits. Step 2 receives that checkpoint
-  (`{ meta, step1TestData, checkpointReceived }`) and returns the `expectedRowIds` a query
-  FROM it must surface. Mutating data between the steps is one way to produce step-2 rows —
-  but not required: on APIs where you can't create rows in-test, pick a step-1 checkpoint far
-  enough back that existing rows fall on both sides (remember your connector's overlap window
-  re-surfaces recent rows in step 2 — include them in `expectedRowIds`).
+- `checkpointFilterStep1Prepare` / `Step2Prepare` — the two-step round-trip test of your
+  differential contract; see [Checkpoint testing](#checkpoint-testing-what-the-two-steps-prove)
+  below, it is the most-misimplemented pair.
 - `matchFilterPrepare` — returns `MatchData` including `expectedMatchRowIds`.
 - `relatedFilterPrepare({ fromObjectMeta, toObjectMeta, fromRelationshipId })` —
   `fromObjectMeta` is the object where the relationship is DECLARED (the parent, in the
@@ -128,13 +123,53 @@ To disable those tests use `cannotTestReason` instead
   object; the harness fetches it via `idsFilter`, then queries the related object with
   `relatedFilter` and compares against `expectedRelatedRowIds`. The test spec side
   (`relationshipsTo`) also lives on the declaring object's features.
-- `upsertPrepare` — returns `UpsertData`; use `withIssueSimulators` / `UpsertIssueSimulator`
-  (`verifyFields`, expected issue `type` + `fatal`) to deliberately exercise your
-  `upsertClean` issue reporting.
+- `upsertPrepare` — returns `UpsertData`; every `verifyFields` path must appear with a
+  non-null (and ideally CHANGED) value in the `update` payload too — see the `verifyFields`
+  contract under "What each test needs" below. Use `withIssueSimulators` /
+  `UpsertIssueSimulator` (`verifyFields`, expected issue `type` + `fatal`) to deliberately
+  exercise your `upsertClean` issue reporting.
 - delete features support soft-delete verification — `isDeleted: { path, valueWhenDeleted }`
   checks a flag field instead of row absence, and `afterDeleteMaxIndexWaitTimeMs` tolerates
   APIs whose deletions surface asynchronously.
 
+## Checkpoint testing: what the two steps prove
+
+The checkpoint contract is a ROUND-TRIP ([05](./05-query.md)): the platform stores whatever
+checkpoint string your final page emits and hands it back on the next differential run. One
+query can't test a round-trip — that's why there are two steps, and they prove different
+things:
+
+- **Step 1 proves your filter reads correctly**: given a checkpoint, the query returns
+  exactly the changed rows.
+- **Step 2 proves the checkpoint you EMIT is usable**: the harness takes the cursor your
+  step-1 query returned — not one you craft — and queries FROM it. This is the step that
+  catches the real bugs: an emitted cursor the query can't parse back, a `Date.now()`
+  checkpoint with no overlap (rows modified mid-query lost forever), a cursor in the wrong
+  timezone/format.
+
+Mechanics: `checkpointFilterStep1Prepare({ meta })` returns
+`{ nextCheckpoint, expectedRowIds }` — your chosen STARTING checkpoint and every row a query
+from it must return. The harness runs the query, verifies the rows, and captures the
+checkpoint your connector emitted. `checkpointFilterStep2Prepare({ meta, step1TestData,
+checkpointReceived })` then returns the `expectedRowIds` for a query from
+`checkpointReceived`.
+
+Three rules that decide whether your expectations are right:
+
+1. **The comparison is an exact set match, both steps** — a missing expected row fails AND
+   an unexpected returned row fails. Never guess; when in doubt, run the same
+   `checkpointFilter` query inside the prepare hook and report what it returns (the worked
+   hooks in [15](./15-example-test-file.md) do exactly this — a shared sandbox is never
+   quiet).
+2. **Your own overlap window works against you in step 2.** You emit checkpoints
+   `startTime - overlap` (05), so a query from `checkpointReceived` legitimately re-returns
+   step-1 rows modified inside the window. Expect them — the classic step-2 failure is
+   listing only the fresh mutation and being surprised by "Returned unexpected row".
+3. **Mutating data between the steps is optional.** Creating one fresh row step 2 must
+   surface is the strongest test, but on APIs where you can't create rows, a self-consistent
+   expectation (query from `checkpointReceived`, report what came back) is valid — the
+   round-trip and overlap behavior still get proven.
+
 ## What each test needs from your sandbox data
 
 Requirements that only surface as cryptic failures when unmet:
@@ -144,14 +179,26 @@ Requirements that only surface as cryptic failures when unmet:
   one row (or no list support and no prepare hook) fails with `"No test data"`. Each id is
   then queried alone and must return exactly that one row.
 - **The upsert test is insert THEN update, twice over.** `upsertPrepare` is called twice; both
-  rows are inserted, re-queried by `idsFilter` and compared on `verifyFields` (which must be
-  non-null on the payload), then BOTH are updated via the declared `upsert.key` and re-queried
-  again. Objects whose rows can never update declare `upsert.insertOnly` — see below.
+  rows are inserted, re-queried by `idsFilter` and compared on `verifyFields`, then BOTH are
+  updated via the declared `upsert.key` and re-queried again. Objects whose rows can never
+  update declare `upsert.insertOnly` — see below.
+- **`verifyFields` is a contract on BOTH payloads.** After each write the harness re-queries
+  the row and compares every `verifyFields` path against the payload it just sent — so each
+  listed field must be present with a non-null value in the `insert` payload AND in the
+  `update` payload. An update payload that omits a verify field fails with
+  `"verifyField value on the upsert row was blank"` (the classic mistake: updating one field
+  while `verifyFields` names another). Give the update DIFFERENT values than the insert —
+  the comparison is payload-vs-queried-row, so unchanged values would let a silently ignored
+  update pass; mutating is what makes the update phase prove anything. Values must round-trip
+  exactly (string-equal after re-query) — pick fields the API echoes verbatim, not ones it
+  normalizes or formats. And return the SAME `verifyFields` paths from both prepare calls:
+  the harness reads row 2's values through call 1's paths.
 - **relatedFilterPrepare needs a declaring-object row that actually has related rows** —
   `expectedRelatedRowIds` must be non-empty. In the parent-owns-children shape that means a
   parent with at least one child in the sandbox (create one in `before()` when the API allows;
   otherwise seed the sandbox account once and document it in the test file).
-- **Checkpoint step 2 does not have to create data** — see the hook notes above.
+- **Checkpoint step 2 does not have to create data** — see
+  [Checkpoint testing](#checkpoint-testing-what-the-two-steps-prove) above.
 
 ## Insert-only / immutable objects
 

package/docs/15-example-test-file.md

@@ -403,7 +403,9 @@ export default class TestFeatureProvider {
         const update = { domain: `${uid}-updated.example.com` };
         result.insert = insert;
         result.update = update;
-        // the harness re-queries the row after each write and compares these paths
+        // the harness re-queries the row after EACH write and compares these paths against
+        // the payload it just sent - so every verify field must be present (non-null) in
+        // BOTH insert and update, and the update should carry a CHANGED value (09)
         result.verifyFields.push([{ path: "domain" }]);
         break;
       }

package/docs/17-user-help-pages.md

@@ -1,9 +1,23 @@
 # Writing the connector's user help pages
 
-A finished connector ships with **customer-facing help pages**. They live in the platform
-repo under `docs/help-markdown/connections/<slug>/` (slug = kebab-case product name, e.g.
-`epicor-prophet-21`) and follow a fixed house pattern — copy the structure of an existing
-current page (`greenhouse/` is a clean modern example) rather than inventing one:
+A finished connector ships with **customer-facing help pages** — and they ship WITH the
+connector, in the script tree, versioned alongside the code they describe:
+
+```
+Connectors/<Name>/help/index.md        (sidecar type: "markdown")
+Connectors/<Name>/help/scripting.md    (sidecar type: "markdown")
+Connectors/<Name>/help/guides/*.md     (optional deep dives)
+```
+
+Author them in the workspace like any other file (`sm push` creates/updates them; each needs
+a sidecar with `type: "markdown"`, `runnable: false`). They are documentation, not code —
+never listed in `module_script_file_paths`, never staged at runtime. When the connector is
+published as a bundle, the platform promotes these pages into the app help under
+`connections/<slug>/…` (promotion is rolling out — until then the docs team lifts them into
+the help repo verbatim, which is why following the house pattern below matters).
+
+The house pattern — copy the structure of an existing current page (`greenhouse/` in the
+platform help is a clean modern example) rather than inventing one:
 
 | File           | Audience  | Content                                                             |
 | -------------- | --------- | -------------------------------------------------------------------- |

package/docs/connector-authoring.md

@@ -36,7 +36,7 @@ queryable.
 | [14-vendor-sdks-and-non-http.md](./14-vendor-sdks-and-non-http.md) | vendor npm SDKs (googleapis, AWS, box), database/on-prem `agent_mode: "always"` connectors, SOAP/XML, self-managed rate limiting |
 | [15-example-test-file.md](./15-example-test-file.md)             | **the canonical worked test file** — the complete `mod Test.mjs` + `mod ObjectTestFeatures.mjs` pair for the Acme connector of 12 (TestFeatureProvider, prepare/cleanup hooks, issue simulators, feature scoping) |
 | [16-the-end-game-syncs.md](./16-the-end-game-syncs.md)           | **why any of this exists** — how the sync pipeline consumes each connector capability, and what silently degrades when one is missing. Read once before designing `meta()` |
-| [17-user-help-pages.md](./17-user-help-pages.md)                 | writing the connector's customer help pages (`connections/<slug>/index.md` + `scripting.md`) — the house structure, frontmatter, and example conventions |
+| [17-user-help-pages.md](./17-user-help-pages.md)                 | writing the connector's customer help pages — they ship IN the connector (`Connectors/<Name>/help/*.md`, sidecar type `"markdown"`); the house structure, frontmatter, and example conventions |
 
 ## The 60-second version
 
@@ -169,8 +169,10 @@ Implementing more methods unlocks more platform capability:
 9. **Run it**: `sm test` = the full suite (identical to the web UI's connector test — including
    write features, so use a sandbox connection); `--only connection|meta|query` narrows. A
    passing `--only` run is NOT suite coverage (09).
-10. **Document it**: write the customer help pages — `connections/<slug>/index.md` +
-    `scripting.md` in the platform repo, derived from `meta()` and the sidecar (17).
+10. **Document it**: write the customer help pages IN the connector —
+    `Connectors/<Name>/help/index.md` + `help/scripting.md` (sidecar type `"markdown"`),
+    derived from `meta()` and the sidecar; they version with the code and publish with the
+    bundle (17).
 
 ## Support
 

package/module.d.ts

@@ -3,7 +3,7 @@
  * SDK specific elements that would be of no interest to users of the API.
  */
 export { EventScriptComplete } from "@syncmatters/script-api";
-export type { Cache, FileProvider, HttpRequest, HttpClientOptions, KvCollection, KvIterator, Logger, EventContext, TestResult, Row, ObjectMeta, ObjectField, ObjectFieldOption, RateLimiterOptions, ScriptType, JsonValuePath, JsonValuePathPart, QueryMatchFilter, QueryCheckpointFilter, EventTypeMeta, ObjectIndex, ObjectMetaUpsert, ObjectRowFilter, ObjectRelationship, ObjectFieldConstraints, ObjectSettingMeta, UpsertIssue, UpsertFieldOptionMeta, RowMatchRuleType, SyncSession, SyncOptimalBatchSizeOperation, SyncOptimalBatchSizeBatchType, } from "@syncmatters/script-api";
+export type { Cache, FileProvider, HttpRequest, HttpClientOptions, KvCollection, KvIterator, Logger, EventContext, TestResult, Row, ObjectMeta, ObjectField, ObjectFieldOption, RateLimiterOptions, ScriptType, JsonValuePath, JsonValuePathPart, QueryMatchFilter, QueryCheckpointFilter, EventTypeMeta, ObjectIndex, ObjectMetaUpsert, ObjectRowFilter, ObjectRelationship, ObjectFieldConstraints, ObjectSettingMeta, UpsertIssue, UpsertFieldOptionMeta, RowMatchRuleType, RowMatchData, RowMatchRuleOrder, SyncSession, SyncOptimalBatchSizeOperation, SyncOptimalBatchSizeBatchType, } from "@syncmatters/script-api";
 export * from "./lib/connector.js";
 export * from "./lib/connector-error.js";
 export { envSet } from "./lib/environment.js";

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@syncmatters/connector-sdk",
-  "version": "1.0.5",
+  "version": "1.0.7",
   "description": "TypeScript type definitions for the SyncMatters connector SDK (types only - connectors execute on the SyncMatters platform)",
   "types": "./index.d.ts",
   "exports": {
@@ -12,9 +12,9 @@
   "license": "MIT",
   "author": "SyncMatters",
   "homepage": "https://syncmatters.com",
-  "typesContentHash": "f0f72980b64b81d6019dbfd86756b2c4dc6ff60eb44b4d1824bde2f8a689fd6b",
+  "typesContentHash": "ae459fd0936deb05ec10d5b85156b8a268d06d5568f1c94d2f81d97c1224efac",
   "dependencies": {
     "@types/node": "*",
-    "@syncmatters/script-api": "^1.0.4"
+    "@syncmatters/script-api": "^1.0.6"
   }
 }