@pixlcore/xycli 1.0.9 → 1.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +4 -1
- package/docs/help.md +8 -6
- package/docs/sync.md +38 -15
- package/docs/two-way-delete-tutorial.md +259 -0
- package/docs/two-way-git-tutorial.md +3 -1
- package/lib/sync-buckets.js +35 -1
- package/lib/sync.js +112 -11
- package/package.json +1 -1
- package/test/README.md +3 -3
- package/test/sync-buckets-unit.test.js +31 -0
- package/test/sync-buckets.test.js +36 -0
- package/test/sync-unit.test.js +115 -7
- package/test/sync.test.js +42 -0
- package/test/tickets.test.js +4 -1
package/README.md
CHANGED
|
@@ -27,6 +27,7 @@ Browse the [screenshot gallery](docs/screenshots.md) for more views, or jump to
|
|
|
27
27
|
- [Global CLI switches](#global-cli-switches)
|
|
28
28
|
- [Tutorial: your first Event, job, and Web Hook](docs/tutorial.md)
|
|
29
29
|
- [Tutorial: two-way sync with Git](docs/two-way-git-tutorial.md)
|
|
30
|
+
- [Tutorial: two-way sync with event-driven deletes](docs/two-way-delete-tutorial.md)
|
|
30
31
|
- [Explore and manage xyOps](#explore-and-manage-xyops)
|
|
31
32
|
- [Transfer data between systems](#transfer-data-between-systems)
|
|
32
33
|
- [Sync your automation with local files](#sync-your-automation-with-local-files)
|
|
@@ -513,10 +514,12 @@ The push above updates xyOps immediately. Remove `--dry` from the pull command t
|
|
|
513
514
|
|
|
514
515
|
Sync supports Alerts, API Keys, Categories, Channels, Events and workflows, Groups, Monitors, Plugins, Tags, and Web Hooks. It updates existing objects and files; use creation commands, imports, setup, or exports to introduce new ones. Two-way sync is experimental and uses modification times, so one-way sync is a better fit for a Git-based source of truth.
|
|
515
516
|
|
|
516
|
-
Delete mode
|
|
517
|
+
Delete mode works in one direction per run. Up-only deletion removes eligible xyOps objects missing from your local inventory; down-only deletion removes local XYPDF definitions and Bucket files missing from xyOps. Objects with a `stock` or `marketplace` property are protected from deletion. Keep a complete inventory on the authoritative side for each selected type, and review a dry run before applying it. Sync applies changes without a confirmation step unless you use `--dry`.
|
|
517
518
|
|
|
518
519
|
Follow the [Two-Way Git Sync Tutorial](docs/two-way-git-tutorial.md) to connect one persistent checkout to a shared repository, commit downloaded changes, discover new Events, and schedule the complete workflow with cron or xySat.
|
|
519
520
|
|
|
521
|
+
For an event-driven workflow that deletes in both directions, creates new definitions only in xyOps, and uses no recurring sync schedule, follow the [Two-Way Git Sync with Event-Driven Deletes Tutorial](docs/two-way-delete-tutorial.md).
|
|
522
|
+
|
|
520
523
|
Read the dedicated [Sync Guide](docs/sync.md) for setup, file layouts, every option, saved defaults, two-way sync, deletion, notifications, and automation with cron, Git hooks, and GitHub Actions.
|
|
521
524
|
|
|
522
525
|
- **Command reference:** [Sync commands](docs/help.md#sync) and [Sync setup](docs/help.md#sync-setup).
|
package/docs/help.md
CHANGED
|
@@ -188,7 +188,7 @@ Your configured permissions still apply during import. If an item fails, import
|
|
|
188
188
|
|
|
189
189
|
## sync
|
|
190
190
|
|
|
191
|
-
For setup, configuration, file layouts, and automation examples, see the dedicated [Filesystem Sync Guide](sync.md).
|
|
191
|
+
For setup, configuration, file layouts, and automation examples, see the dedicated [Filesystem Sync Guide](sync.md). For separate Git- and xyOps-triggered delete passes, see the [event-driven two-way delete tutorial](two-way-delete-tutorial.md).
|
|
192
192
|
|
|
193
193
|
Sync existing xyOps definitions with a directory of XYPDF files. Each run scans the directory and its subdirectories, shows any differences, and applies changes in the directions you select. **Changes are applied immediately unless you add `--dry`; there is no `--confirm` step.**
|
|
194
194
|
|
|
@@ -225,20 +225,22 @@ Repeated sources with the same type and ID generate warnings, even if their cont
|
|
|
225
225
|
|
|
226
226
|
External string properties are loaded automatically from adjacent files named `BASENAME-PROPERTY.EXT`, such as `My-Plugin-script.js` beside `My-Plugin.json`, or `My-Event-params.script.sh` beside `My-Event.json`. The extension is your choice. Keep only one external file per property, and rename its basename along with the JSON file.
|
|
227
227
|
|
|
228
|
-
Bucket metadata uses a normal XYPDF source. For a Bucket titled `My Bucket Name`, its separate JSON data and uploaded files live beside `buckets/My-Bucket-Name.json` in `buckets/My-Bucket-Name/data.json` and `buckets/My-Bucket-Name/files/`. Both `data.json` and `files/` are required for a selected local bucket. Bucket data upsync replaces the entire remote data object. Bucket files are matched by xyOps filename normalization and transferred according to size and modification time. Two-way bucket sync uses the data record and file timestamps from xyOps; a transfer updates local file times to match.
|
|
228
|
+
Bucket metadata uses a normal XYPDF source. For a Bucket titled `My Bucket Name`, its separate JSON data and uploaded files live beside `buckets/My-Bucket-Name.json` in `buckets/My-Bucket-Name/data.json` and `buckets/My-Bucket-Name/files/`. Both `data.json` and `files/` are required for a selected local bucket. Bucket data upsync replaces the entire remote data object. Bucket files are matched by xyOps filename normalization and transferred according to size and modification time. Two-way bucket sync uses the data record and file timestamps from xyOps; a transfer updates local file times to match. Up-only `--delete buckets` removes remote files missing locally; down-only `--delete buckets` removes local files missing remotely.
|
|
229
229
|
|
|
230
230
|
**Delete mode**
|
|
231
231
|
|
|
232
|
-
Add `--delete TYPES` to
|
|
232
|
+
Add `--delete TYPES` to remove definitions missing from the authoritative side of a one-way sync. With `--up` only, it deletes xyOps objects missing from the local tree. With `--down` only, it deletes local XYPDF sources missing from xyOps, plus their external property files. The CLI rejects deletion when both directions or neither direction are enabled, including saved defaults and dry runs. Use `--up false` or `--down false` to disable an inherited direction. See [Delete mode](sync.md#delete-mode) before using it.
|
|
233
233
|
|
|
234
234
|
```sh
|
|
235
235
|
xy sync ./ --up events,plugins --delete events --dry
|
|
236
236
|
xy sync ./ --up events,plugins --delete events
|
|
237
|
+
xy sync ./ --up false --down events,plugins --delete events,plugins --dry
|
|
238
|
+
xy sync ./ --up false --down events,plugins --delete events,plugins
|
|
237
239
|
```
|
|
238
240
|
|
|
239
|
-
**Objects with a `stock` or `marketplace` property are
|
|
241
|
+
**Objects with a `stock` or `marketplace` property are ignored by deletion**, including during dry runs. Upward deletion checks the xyOps definition; downward deletion checks the local XYPDF. Setup inclusion flags cannot override this protection.
|
|
240
242
|
|
|
241
|
-
**The
|
|
243
|
+
**The authoritative side must have a complete inventory of the objects you intend to keep for every type selected for deletion.** Delete mode is not limited to previously synced objects. A missing local file can delete a remote object during upsync; a new local file not yet created in xyOps can be deleted during downsync. Ensure the API Key can see the complete remote inventory, review a dry run, and coordinate separate up and down jobs before enabling automatic deletion. Warnings or errors before the delete phase skip that phase; earlier successful updates remain in place.
|
|
242
244
|
|
|
243
245
|
**System Hook deletions**
|
|
244
246
|
|
|
@@ -265,7 +267,7 @@ xy sync ./ --up events,plugins \
|
|
|
265
267
|
- `--error_email ADDRESS` sends warnings and errors by email through xyOps.
|
|
266
268
|
- `--error_event EVENT_ID` runs an Event on warnings or errors.
|
|
267
269
|
|
|
268
|
-
Completion commands run in a local shell in the first base directory. Successful up-only deletion triggers `up_cmd`;
|
|
270
|
+
Completion commands run in a local shell in the first base directory. Successful up-only deletion triggers `up_cmd`; down-only and System Hook deletions trigger `down_cmd` when it is configured. Event notifications receive `input.data.errors` and `input.data.warnings`.
|
|
269
271
|
|
|
270
272
|
The configured API Key needs the appropriate resource permissions and `update_state` for sync tracking updates. Notification actions also need their usual permissions. Earlier successful changes are not rolled back if a later operation fails. Retry the sync after correcting an error; previously applied updates compare equal, and deletion remains disabled until the pre-delete phase completes without warnings or errors.
|
|
271
273
|
|
package/docs/sync.md
CHANGED
|
@@ -40,10 +40,12 @@ Sync has two directions. **Up** sends local definitions to xyOps. **Down** write
|
|
|
40
40
|
|
|
41
41
|
**Sync applies changes immediately. There is no confirmation step.** Add `--dry` to preview a run before allowing it to change xyOps or your files.
|
|
42
42
|
|
|
43
|
-
Sync updates *changes* to objects that already exist on both sides. It does not create new xyOps objects
|
|
43
|
+
Sync updates *changes* to objects that already exist on both sides. It does not create new xyOps objects or generate local files for newly created objects (except during [setup](#set-up-your-first-sync-tree)). Use resource creation commands or imports to create objects, and setup or exports to introduce their local files. One-way [delete mode](#delete-mode) can remove definitions missing from the authoritative side.
|
|
44
44
|
|
|
45
45
|
For a Git-based deployment workflow, start with up-only sync. A fresh checkout is a good source for an explicit push, but its filesystem timestamps are not reliable evidence for two-way conflict decisions.
|
|
46
46
|
|
|
47
|
+
For two-way deletion driven by accepted Git pushes and xyOps System Activity triggers, see the [event-driven two-way delete tutorial](two-way-delete-tutorial.md).
|
|
48
|
+
|
|
47
49
|
## Supported resources and requirements
|
|
48
50
|
|
|
49
51
|
Install and connect xyCLI as described in [Getting started](../README.md#getting-started). The CLI requires Node.js 20.19.0 or later and xyOps 1.0.96 or later. The machine running sync needs network access to the xyOps API and access to the local directories you select.
|
|
@@ -70,7 +72,7 @@ Secret Vaults, Users, and Roles are not supported by sync. Individual workers, j
|
|
|
70
72
|
|
|
71
73
|
### Permissions
|
|
72
74
|
|
|
73
|
-
Your API Key needs permission to read the definitions you scan and to edit each type you push.
|
|
75
|
+
Your API Key needs permission to read the definitions you scan and to edit each type you push. Up-only delete mode additionally requires the corresponding delete privileges. Down-only deletion removes local files and does not call the remote delete APIs. Resource restrictions, such as Category and Group access, still apply.
|
|
74
76
|
|
|
75
77
|
Sync also maintains remote-management tracking which requires the [update_state](https://docs.xyops.io/privileges/update_state) privilege. A pull or two-way run can update this tracking too. Notification email requires the [send_emails](https://docs.xyops.io/privileges/send_emails) privilege; an error Event requires the [run_jobs](https://docs.xyops.io/privileges/run_jobs) privilege, permission to use that Event's resources, and an enabled manual trigger.
|
|
76
78
|
|
|
@@ -153,7 +155,7 @@ Setup normally omits objects marked as stock or installed from the Marketplace.
|
|
|
153
155
|
xy sync setup all --stock --marketplace --file_props script,params.script
|
|
154
156
|
```
|
|
155
157
|
|
|
156
|
-
These switches are optional export choices, not requirements for delete mode.
|
|
158
|
+
These switches are optional export choices, not requirements for delete mode. Up-only deletion excludes remote objects marked `stock` or `marketplace`. Down-only deletion excludes local sources carrying either property. Leave Marketplace Plugins managed by the Marketplace and its upgrade system; a standard sync tree can focus on definitions you maintain yourself.
|
|
157
159
|
|
|
158
160
|
### Rerun setup carefully
|
|
159
161
|
|
|
@@ -393,9 +395,9 @@ Relative paths resolve against the CLI's current working directory. Completion c
|
|
|
393
395
|
| --- | --- | --- | --- |
|
|
394
396
|
| `--up TYPES` | `sync.up` | Disabled | Sync upwards to xyOps from local sources for these types. |
|
|
395
397
|
| `--down TYPES` | `sync.down` | Disabled | Down downwards to existing local sources from xyOps for these types. |
|
|
396
|
-
| `--delete TYPES` | `sync.delete` | Disabled | Delete
|
|
398
|
+
| `--delete TYPES` | `sync.delete` | Disabled | Delete definitions missing from the sole enabled direction's authoritative side. Requires up-only or down-only sync; see [Delete mode](#delete-mode). |
|
|
397
399
|
| `--syshook ACTION` | n/a | Disabled | Handle one supported definition deletion activity from STDIN as a local-only deletion; see [System Hook deletion](#system-hook-deletion). |
|
|
398
|
-
| `--ignore_missing` | `sync.ignore_missing` | Off | Skip local sources whose type and ID are absent from xyOps, without a warning or failure.
|
|
400
|
+
| `--ignore_missing` | `sync.ignore_missing` | Off | Skip local sources whose type and ID are absent from xyOps, without a warning or failure. Down-only deletion takes precedence for types selected by `--delete`. |
|
|
399
401
|
| `--up_cmd COMMAND` | `sync.up_cmd` | None | Run a local shell command after at least one successful upsync. |
|
|
400
402
|
| `--down_cmd COMMAND` | `sync.down_cmd` | None | Run a local shell command after at least one successful downsync, or an explicit command after a System Hook deletion. |
|
|
401
403
|
| `--cmd_timeout SECONDS` | `sync.cmd_timeout` | `30` | Timeout for each completion command. |
|
|
@@ -453,13 +455,13 @@ For automatic two-way operation, use a persistent tree and run every minute. Bot
|
|
|
453
455
|
|
|
454
456
|
### Remove a definition during two-way sync
|
|
455
457
|
|
|
456
|
-
|
|
458
|
+
One command with both sync directions does not propagate deletions. Keep `--delete false` in a scheduled two-way command, and remove a definition from both sides deliberately:
|
|
457
459
|
|
|
458
460
|
1. Delete the definition in the xyOps web interface first.
|
|
459
461
|
2. Remove its local XYPDF JSON file and any neighboring external property files. If the tree is stored in Git, commit and push their removal so the dedicated sync checkout receives the deletion.
|
|
460
462
|
3. Run sync again, or resume the schedule, after the local files are gone. Without `--ignore_missing`, pause the schedule during these steps if you want to avoid a failed pass between the two removals.
|
|
461
463
|
|
|
462
|
-
If a sync runs after the xyOps deletion but before local cleanup, the leftover source normally produces a missing-object warning and stops the run before any definition changes. Add `--ignore_missing` to ordinary sync and any following `sync setup --new` command to skip that source and keep processing other definitions. This does not remove the local files or propagate the deletion to Git. Remove those files yourself to complete the deletion. Other scan warnings and errors still stop the run.
|
|
464
|
+
If a sync runs after the xyOps deletion but before local cleanup, the leftover source normally produces a missing-object warning and stops the run before any definition changes. Add `--ignore_missing` to ordinary sync and any following `sync setup --new` command to skip that source and keep processing other definitions. This does not remove the local files or propagate the deletion to Git. Remove those files yourself to complete the deletion. Other scan warnings and errors still stop the run. For an event-driven workflow with one authoritative direction per run, see [Delete mode](#delete-mode).
|
|
463
465
|
|
|
464
466
|
Do not remove the Git files first while the definition still exists in xyOps. A scheduled `sync setup --new` pass can export the still-existing definition back into the repository. See the [Two-Way Git Sync Tutorial](two-way-git-tutorial.md#delete-a-definition-from-both-sides) for the Git workflow.
|
|
465
467
|
|
|
@@ -523,9 +525,15 @@ Make sure only one sync instance talks to one xyOps installation.
|
|
|
523
525
|
|
|
524
526
|
## Delete mode
|
|
525
527
|
|
|
526
|
-
Delete mode
|
|
528
|
+
Delete mode removes definitions missing from the authoritative side of a **one-way** sync. With `--up` only, it deletes xyOps definitions that have no local XYPDF source. With `--down` only, it deletes local XYPDF sources that have no matching visible xyOps definition, along with their external property files. Select the affected resource types with `--delete TYPES`. The CLI rejects deletion if both directions, or neither direction, are enabled, including directions inherited from saved configuration. A single two-way sync command cannot use delete mode.
|
|
529
|
+
|
|
530
|
+
The chosen direction applies to every missing definition of the selected types. A newly added Git file that has not yet been imported into xyOps looks the same as a definition deleted in xyOps, so a down-only delete run removes that file. Likewise, an xyOps definition that has not yet been exported looks the same as one deleted from Git to an up-only delete run. Coordinate Git-triggered and xyOps-triggered jobs so they do not overlap or process pending changes out of order. A host-local PID lock cannot serialize jobs on different machines.
|
|
531
|
+
|
|
532
|
+
The [event-driven two-way delete tutorial](two-way-delete-tutorial.md) shows separate authoritative triggers for Git pushes, xyOps updates and deletions, and xyOps creations.
|
|
527
533
|
|
|
528
|
-
###
|
|
534
|
+
### Up-only deletion
|
|
535
|
+
|
|
536
|
+
#### Prepare a complete inventory
|
|
529
537
|
|
|
530
538
|
The scanned directories must contain a source for **every other remote object of each type selected for deletion** that you intend to keep. Delete mode considers all eligible objects in the visible remote collection, not just objects previously synced or carrying a remote-management warning.
|
|
531
539
|
|
|
@@ -541,7 +549,7 @@ xy sync setup events plugins categories
|
|
|
541
549
|
|
|
542
550
|
Verify that every object eligible for deletion that you intend to keep has a corresponding source, including workflows under the `events` selection. Stock and Marketplace objects may be omitted. Avoid filename collisions, omitted resources, missing mounts, and hidden source directories.
|
|
543
551
|
|
|
544
|
-
|
|
552
|
+
#### Preview a deliberate deletion
|
|
545
553
|
|
|
546
554
|
Remove the source file(s) for the Event you intend to delete, then preview from the complete inventory:
|
|
547
555
|
|
|
@@ -559,10 +567,25 @@ Deleting an Event does not automatically remove its dependencies. Remote API rul
|
|
|
559
567
|
|
|
560
568
|
Deletion only begins after every update has completed and the sync-tracking state has been written without warnings or errors. If either phase fails, the entire delete pass is skipped for that run. Earlier successful updates remain in place, so correct the error and retry; definitions already updated will compare equal. Once deletion begins, successful earlier deletions are not rolled back if a later deletion fails.
|
|
561
569
|
|
|
562
|
-
With `--delete buckets`, a remote Bucket file missing from a selected local Bucket's `files/` directory is deleted individually. Removing the Bucket XYPDF source deletes the whole Bucket, including all its data and files. A missing or invalid `data.json`, incomplete file inventory, or failed transfer prevents Bucket file deletion and the whole-object delete pass.
|
|
570
|
+
With up-only `--delete buckets`, a remote Bucket file missing from a selected local Bucket's `files/` directory is deleted individually. Removing the Bucket XYPDF source deletes the whole Bucket, including all its data and files. A missing or invalid `data.json`, incomplete file inventory, or failed transfer prevents Bucket file deletion and the whole-object delete pass.
|
|
563
571
|
|
|
564
572
|
An unavailable base directory stops the scan, but an existing empty directory can look like an empty inventory. Do not treat that guard as protection against an empty mount, accidental file removal, or missing exports of your own definitions.
|
|
565
573
|
|
|
574
|
+
### Down-only deletion
|
|
575
|
+
|
|
576
|
+
Run a down-only pass when xyOps should determine which local definitions remain. Specify the full local sync tree, and use `--up false` if saved configuration would otherwise enable upsync:
|
|
577
|
+
|
|
578
|
+
```sh
|
|
579
|
+
xy sync /srv/xyops-sync --up false --down events,plugins --delete events,plugins --dry
|
|
580
|
+
xy sync /srv/xyops-sync --up false --down events,plugins --delete events,plugins --down_cmd ./commit-downloads.sh
|
|
581
|
+
```
|
|
582
|
+
|
|
583
|
+
Down-only deletion removes matching local XYPDF sources and their validated external property files. It matches by item type and exact ID, not filename. A missing local source is not created by ordinary downsync; run `xy sync setup TYPES --new` separately to export newly created xyOps definitions. Setup runs from the current directory, so run it from the sync tree's root.
|
|
584
|
+
|
|
585
|
+
For Buckets, down-only `--delete buckets` also removes local uploaded files absent from an existing remote Bucket. If the entire Bucket is absent from xyOps, it removes the local Bucket XYPDF, `data.json`, and uploaded files, then removes the empty content directories. Unexpected content in a Bucket directory stops the whole-object local delete pass. Stock and Marketplace markers in a local XYPDF exclude that definition from deletion.
|
|
586
|
+
|
|
587
|
+
The local delete pass starts only after the source scan, definition downloads, sync-state update, and Bucket content pass finish without warnings or errors. It checks every candidate before removing any whole-object files. A successful local deletion triggers `down_cmd`, including when there was no ordinary download. If a later file removal fails, earlier removals remain and the run exits nonzero; inspect the tree and retry.
|
|
588
|
+
|
|
566
589
|
Keep deletion disabled in general-purpose cron, Git hooks, and CI until you have a deliberate policy for maintaining and checking the complete inventory. Disable saved deletion for other runs with `--delete false`.
|
|
567
590
|
|
|
568
591
|
It is also highly recommended that you [keep daily backups of critical data](https://docs.xyops.io/hosting/daily-backups) before enabling sync, especially sync with delete.
|
|
@@ -585,7 +608,7 @@ xy sync ./ --down events,plugins --down_cmd "./commit-downloads.sh" --cmd_timeou
|
|
|
585
608
|
### When commands run
|
|
586
609
|
|
|
587
610
|
- `up_cmd` runs if at least one object was successfully upsynced (this includes deletes).
|
|
588
|
-
- `down_cmd` runs if at least one object was successfully
|
|
611
|
+
- `down_cmd` runs if at least one object was successfully downloaded or locally deleted by down-only sync. In `--syshook` mode, an explicitly supplied `down_cmd` runs after the matching local files are deleted.
|
|
589
612
|
- During setup, an explicitly supplied `down_cmd` runs if at least one definition was written.
|
|
590
613
|
- If both apply, `up_cmd` runs first, then `down_cmd`.
|
|
591
614
|
- Neither runs for an identical/no-op run, a scan that stopped before changes, or a dry run.
|
|
@@ -978,13 +1001,13 @@ Repository concurrency controls these workflow runs. Coordinate all writers to t
|
|
|
978
1001
|
| The wrong directory is scanned | Positional paths take precedence, followed by `base_dirs`, then the current directory. Check your command and config; use absolute paths in automation. |
|
|
979
1002
|
| A config override seems to vanish | The user file replaces a system `sync` object as a whole. Environment settings take precedence over files. See [configuration](#configuration-and-option-reference). |
|
|
980
1003
|
| An object is missing from the local tree | Ordinary downsync does not discover it. Export that object or run setup in a separate staging directory. |
|
|
981
|
-
| A source cannot find its remote object | Match its exact `type` and `data.id` to an existing visible object in this xyOps installation. Create/import missing objects separately. |
|
|
1004
|
+
| A source cannot find its remote object | Match its exact `type` and `data.id` to an existing visible object in this xyOps installation. Create/import missing objects separately, or use down-only delete mode if xyOps is authoritative and the local source should be removed. |
|
|
982
1005
|
| A duplicate-source warning stops sync | Keep one file per resource type and ID, even when the copies are identical. Remove duplicates and avoid overlapping base directories, then retry. |
|
|
983
1006
|
| One bad file stops unrelated updates | The scan validates all JSON sources before applying changes. Fix malformed JSON, multi-item XYPDF, unsupported types, missing objects, or unreadable property files. |
|
|
984
1007
|
| A script edit has no effect | Check the matching JSON stem, property suffix, filename extension, and property path. Hidden paths are skipped. |
|
|
985
1008
|
| Two-way sync overwrites an unexpected side | Compare the remote modification time with the newest JSON/property-file mtime. Fresh checkouts, clock skew, or touched files can make local sources appear newer. |
|
|
986
1009
|
| An Event or workflow does not execute | Sync saves definitions; it does not run them. Use `xy run EVENT_ID` separately. |
|
|
987
|
-
| `down_cmd` does not run | Normal sync requires a successful download. Setup requires at least one written definition and an explicit `--down_cmd`. No-op and dry runs do not trigger it. |
|
|
1010
|
+
| `down_cmd` does not run | Normal sync requires a successful download or down-only deletion. Setup requires at least one written definition and an explicit `--down_cmd`. No-op and dry runs do not trigger it. |
|
|
988
1011
|
| A download commit launches another sync | Disable deployment Git hooks for commits made by `down_cmd`. The nested sync will otherwise be rejected by the built-in PID lock. |
|
|
989
1012
|
| Sync reports another PID is running | Another local sync owns the lock. Let it finish. If the PID is no longer alive, the next run recovers the stale file automatically. If the PID belongs to an unrelated recycled process, verify that no sync is active before removing the reported lock file. |
|
|
990
1013
|
| A scheduled run prints nothing | `--quiet` suppresses routine output. Warnings and errors still print to standard error. Check that the scheduler captures it. |
|
|
@@ -1002,7 +1025,7 @@ Before important deployments, review a dry run, retain recoverable versions of b
|
|
|
1002
1025
|
|
|
1003
1026
|
Sync uses IDs rather than titles or filenames, and does not remap IDs for a different xyOps installation. Duplicate sources for the same type and ID generate warnings and stop the run before changes; keep one source per object and avoid overlapping base directories. Split inventories by ownership when useful, but remember that each run replaces the shared up-only remote-management tracking map rather than combining independent jobs' maps.
|
|
1004
1027
|
|
|
1005
|
-
New
|
|
1028
|
+
New definitions still require explicit creation or setup export. Deletion requires separate one-way runs with a complete inventory on the authoritative side. Two-way sync remains experimental and resolves existing whole objects by modification time, without a merge or conflict prompt.
|
|
1006
1029
|
|
|
1007
1030
|
- **Command reference:** [Sync](help.md#sync), [Sync setup](help.md#sync-setup), [Exports](help.md#export), [Imports](help.md#import), and [Run](help.md#run).
|
|
1008
1031
|
- **Learn more about xyOps:** [xyOps Portable Data Format](https://docs.xyops.io/xypdf) and [xyOps API Keys](https://docs.xyops.io/api#api-keys).
|
|
@@ -0,0 +1,259 @@
|
|
|
1
|
+
# Two-Way Git Sync with Event-Driven Deletes
|
|
2
|
+
|
|
3
|
+
This tutorial keeps Events, workflows, and Plugins in both xyOps and Git. Existing definitions can be edited on either side, and deletions can start on either side. New definitions are created in the xyOps UI and exported to Git. Every sync runs because Git accepted a push or xyOps recorded an activity. There are no cron jobs or recurring sync schedules.
|
|
4
|
+
|
|
5
|
+
Use xyCLI version **v1.1.0+** (with down-sync delete support) and xyOps version **v1.1.3+** (with System Activity triggers). Both features must be available before you enable this workflow.
|
|
6
|
+
|
|
7
|
+
> [!IMPORTANT]
|
|
8
|
+
> **Two-way deletion requires authoritative triggers in both directions.** Run `--up --delete` only in response to an accepted Git change, and run `--down --delete` only in response to xyOps activity. Each invocation has exactly one authoritative side. A single command with both `--up` and `--down` cannot use `--delete`.
|
|
9
|
+
|
|
10
|
+
| Authoritative action | Trigger | Command | Result |
|
|
11
|
+
| --- | --- | --- | --- |
|
|
12
|
+
| Git accepts a contributor push | Git `post-receive` hook or GitHub `push` webhook | `xy sync ... --up events,plugins --down false --delete events,plugins` | Git updates existing definitions and removes missing ones from xyOps. |
|
|
13
|
+
| xyOps updates or deletes an Event or Plugin | **System Activity** Event | `xy sync ... --up false --down events,plugins --delete events,plugins` | xyOps updates existing files and removes missing XYPDF files and their owned neighbors. |
|
|
14
|
+
| xyOps creates an Event or Plugin | Separate **System Activity** Event | `xy sync setup events plugins --new ...` | New definitions are exported, committed, and pushed to Git. |
|
|
15
|
+
|
|
16
|
+
> [!CAUTION]
|
|
17
|
+
> There is a potential race condition between creating an item in xyOps and exporting it to Git. If a Git-triggered up-delete runs before the new XYPDF file reaches the checkout, it can delete that newly created item from xyOps. The shared job queue prevents runs from overlapping, but it cannot guarantee which trigger arrives first. See [Ordering and recovery](#ordering-and-recovery).
|
|
18
|
+
|
|
19
|
+
The examples in this tutorial include **Events** (which includes Workflows) and **Plugins**. To manage other resource types, add them to the commands and choose their corresponding activity types.
|
|
20
|
+
|
|
21
|
+
## Before you begin
|
|
22
|
+
|
|
23
|
+
Use one persistent Git checkout on a machine running xySat. Target all three xyOps Events below to that machine, where the checkout, xyCLI configuration, and Git credentials are available. The xyOps conductor does not need a copy of the repository. Use one operating-system account for all three jobs.
|
|
24
|
+
|
|
25
|
+
You will need:
|
|
26
|
+
|
|
27
|
+
- A xyOps API Key that can see the **complete** Event and Plugin inventories, update and delete those types, and update global sync state. A restricted or partial inventory is unsafe for up-only deletion.
|
|
28
|
+
- A dedicated repository and a persistent checkout, such as `/srv/xyops-automation`, with noninteractive Git pull and push access.
|
|
29
|
+
- A Git author identity for commits generated by xyOps.
|
|
30
|
+
- Shared xyOps job capacity and queue limits on all three Events.
|
|
31
|
+
- A way to monitor failed xyOps jobs and Git webhook deliveries, then retry them after checking both sides.
|
|
32
|
+
|
|
33
|
+
This is a full-inventory workflow. Do not point it at a partial export, empty mount, fresh clone missing files, or API Key with an incomplete view. Stock and Marketplace definitions are naturally excluded.
|
|
34
|
+
|
|
35
|
+
## 1. Prepare the checkout
|
|
36
|
+
|
|
37
|
+
Clone the repository on the sync worker, then configure the account that will run the jobs:
|
|
38
|
+
|
|
39
|
+
```sh
|
|
40
|
+
git clone git@github.com:YOUR_ORG/xyops-automation.git /srv/xyops-automation
|
|
41
|
+
cd /srv/xyops-automation
|
|
42
|
+
git config user.name "xyOps Sync"
|
|
43
|
+
git config user.email "xyops-sync@example.com"
|
|
44
|
+
xy config --base_url https://xyops.example.com --api_key YOUR_API_KEY
|
|
45
|
+
xy dashboard
|
|
46
|
+
```
|
|
47
|
+
|
|
48
|
+
Keep the API Key and Git credentials outside the repository. Use `main` as the dedicated branch in the examples below, or replace it consistently with your branch name.
|
|
49
|
+
|
|
50
|
+
Export a complete starter inventory from the repository root. `--file_props` keeps common Plugin and Event scripts in neighboring files:
|
|
51
|
+
|
|
52
|
+
```sh
|
|
53
|
+
xy sync setup events plugins --file_props script,params.script --dry
|
|
54
|
+
xy sync setup events plugins --file_props script,params.script
|
|
55
|
+
git status --short
|
|
56
|
+
```
|
|
57
|
+
|
|
58
|
+
Review the generated files, including scripts and potentially sensitive configuration. Confirm that every non-stock Event, workflow, and Plugin visible to the sync API Key is present. Then commit and push the initial tree:
|
|
59
|
+
|
|
60
|
+
```sh
|
|
61
|
+
git add --all
|
|
62
|
+
git commit -m "Export xyOps automation"
|
|
63
|
+
git push -u origin main
|
|
64
|
+
```
|
|
65
|
+
|
|
66
|
+
Create the automation Events described below only after this inventory exists. Export them before enabling the Git trigger, so an early up-only delete pass cannot mistake an automation Event for a Git deletion.
|
|
67
|
+
|
|
68
|
+
## 2. Add a commit helper
|
|
69
|
+
|
|
70
|
+
Save this script as `/usr/local/sbin/commit-xyops-sync` on the sync worker. Both xyOps-originated jobs will use it as their explicit `--down_cmd`, to run after local changes have been made:
|
|
71
|
+
|
|
72
|
+
```sh
|
|
73
|
+
#!/bin/sh
|
|
74
|
+
set -eu
|
|
75
|
+
|
|
76
|
+
cd /srv/xyops-automation
|
|
77
|
+
|
|
78
|
+
# This checkout is dedicated to xyOps automation. Stage removals as well as
|
|
79
|
+
# new and updated files, then avoid an empty commit if nothing changed in Git.
|
|
80
|
+
git add --all
|
|
81
|
+
if git diff --cached --quiet; then
|
|
82
|
+
exit 0
|
|
83
|
+
fi
|
|
84
|
+
|
|
85
|
+
# The Git trigger should ignore commits carrying this prefix. They came from
|
|
86
|
+
# xyOps and must not cause a second authoritative up-delete run.
|
|
87
|
+
git commit -m "[xyops-sync] Export xyOps changes"
|
|
88
|
+
git push origin HEAD:main
|
|
89
|
+
```
|
|
90
|
+
|
|
91
|
+
Make it executable with `chmod 755 /usr/local/sbin/commit-xyops-sync`. This helper assumes the repository contains only the automation tree. If it contains unrelated work, narrow the staged paths and review the resulting commit before using it unattended.
|
|
92
|
+
|
|
93
|
+
## 3. Add one sync script for the three jobs
|
|
94
|
+
|
|
95
|
+
Save this as `/usr/local/sbin/xyops-git-sync` on the sync worker:
|
|
96
|
+
|
|
97
|
+
```sh
|
|
98
|
+
#!/bin/sh
|
|
99
|
+
set -eu
|
|
100
|
+
|
|
101
|
+
# Include the directories containing Node.js, xyCLI, and Git on this worker.
|
|
102
|
+
PATH="/usr/local/bin:/usr/bin:/bin"
|
|
103
|
+
export PATH
|
|
104
|
+
|
|
105
|
+
REPO_DIR="/srv/xyops-automation"
|
|
106
|
+
XY_BIN="/usr/local/bin/xy"
|
|
107
|
+
COMMIT_CMD="/usr/local/sbin/commit-xyops-sync"
|
|
108
|
+
SYNC_TYPES="events,plugins"
|
|
109
|
+
MODE="${1:-}"
|
|
110
|
+
|
|
111
|
+
cd "$REPO_DIR"
|
|
112
|
+
|
|
113
|
+
# Work only from the dedicated branch and a clean, existing checkout. A
|
|
114
|
+
# missing tree or leftover change must never be treated as a deletion.
|
|
115
|
+
if [ "$(git symbolic-ref --quiet --short HEAD)" != main ]; then
|
|
116
|
+
echo "Sync stopped: checkout is not on main." >&2
|
|
117
|
+
exit 1
|
|
118
|
+
fi
|
|
119
|
+
if [ -n "$(git status --porcelain)" ]; then
|
|
120
|
+
echo "Sync stopped: checkout has uncommitted changes." >&2
|
|
121
|
+
exit 1
|
|
122
|
+
fi
|
|
123
|
+
if [ ! -d events ] || [ ! -d plugins ]; then
|
|
124
|
+
echo "Sync stopped: Event or Plugin inventory directory is missing." >&2
|
|
125
|
+
exit 1
|
|
126
|
+
fi
|
|
127
|
+
|
|
128
|
+
# Pull accepted Git changes before scanning. A failed fast-forward requires
|
|
129
|
+
# review rather than a reset that could discard a xyOps-generated commit.
|
|
130
|
+
git pull --quiet --ff-only origin main
|
|
131
|
+
|
|
132
|
+
case "$MODE" in
|
|
133
|
+
up)
|
|
134
|
+
"$XY_BIN" sync "$REPO_DIR" \
|
|
135
|
+
--up "$SYNC_TYPES" --down false --delete "$SYNC_TYPES" \
|
|
136
|
+
--up_cmd false --down_cmd false
|
|
137
|
+
;;
|
|
138
|
+
|
|
139
|
+
down)
|
|
140
|
+
"$XY_BIN" sync "$REPO_DIR" \
|
|
141
|
+
--up false --down "$SYNC_TYPES" --delete "$SYNC_TYPES" \
|
|
142
|
+
--down_cmd "$COMMIT_CMD" --cmd_timeout 120
|
|
143
|
+
;;
|
|
144
|
+
|
|
145
|
+
create)
|
|
146
|
+
# Setup scans the current directory and adds only remote definitions
|
|
147
|
+
# whose type and ID are not already represented in the checkout.
|
|
148
|
+
"$XY_BIN" sync --setup "$SYNC_TYPES" --new \
|
|
149
|
+
--file_props script,params.script \
|
|
150
|
+
--down_cmd "$COMMIT_CMD" --cmd_timeout 120
|
|
151
|
+
;;
|
|
152
|
+
|
|
153
|
+
*)
|
|
154
|
+
echo "Usage: $0 up|down|create" >&2
|
|
155
|
+
exit 2
|
|
156
|
+
;;
|
|
157
|
+
esac
|
|
158
|
+
```
|
|
159
|
+
|
|
160
|
+
Make it executable with `chmod 755 /usr/local/sbin/xyops-git-sync`. Change the paths and `PATH` for your worker. The directory checks are a basic guard against a missing checkout. They do not prove that the inventory is complete, so inspect it before enabling deletion and after any Git or API failure.
|
|
161
|
+
|
|
162
|
+
Set a suitable job timeout for Git network operations. All three Events will share a xyOps capacity pool so only one can use this checkout at a time.
|
|
163
|
+
|
|
164
|
+
## 4. Configure the xyOps Activity Events
|
|
165
|
+
|
|
166
|
+
Create three Shell Plugin Events in xyOps, initially disabled. Target only the sync worker and give each Event the indicated trigger and script. The [System Activity trigger](https://docs.xyops.io/triggers#activity) lets you select multiple activity types in the UI; it launches a job when any of them occur. Note that the xyOps scheduler must be enabled for Activity triggers to run.
|
|
167
|
+
|
|
168
|
+
| Event | Trigger | Select these activities | Shell script body |
|
|
169
|
+
| --- | --- | --- | --- |
|
|
170
|
+
| **Sync deletions and updates from xyOps** | System Activity | `event_update`, `plugin_update`, `event_delete`, `plugin_delete` | `/usr/local/sbin/xyops-git-sync down` |
|
|
171
|
+
| **Export creations from xyOps** | System Activity | `event_create`, `plugin_create` | `/usr/local/sbin/xyops-git-sync create` |
|
|
172
|
+
| **Apply accepted Git pushes** | Manual | No activity selection | `/usr/local/sbin/xyops-git-sync up` |
|
|
173
|
+
|
|
174
|
+
The update activities make xyOps UI edits flow back to Git too. The separate creation Event runs `sync setup --new`, because ordinary downsync updates existing local sources but does not create files for new xyOps definitions. The setup command's `--down_cmd` runs only when it writes files; the down-delete command's `--down_cmd` runs only after local changes. Neither creates an empty Git commit on a no-op run.
|
|
175
|
+
|
|
176
|
+
The Git Event has an enabled **Manual** trigger so a Git hook or verified webhook receiver can launch it with `xy run` or the [`run_event` API](https://docs.xyops.io/api#run_event). It has no schedule. The Git push is still the authoritative reason for that run.
|
|
177
|
+
|
|
178
|
+
On **each** of these three Events, set a **[Max Jobs Limit](https://docs.xyops.io/#Docs/limits/max-jobs-limit)** of `1` with the same **[Shared Capacity Key](https://docs.xyops.io/#Docs/limits/shared-capacity-key)**, such as `xyops_git_sync`. Also set a **[Max Queue Limit](https://docs.xyops.io/#Docs/limits/max-queue-limit)** with the same positive amount on each Event, sized for the bursts you expect. This creates one shared, FIFO job queue for all sync operations. Without queue capacity, a trigger arriving while another sync job runs may be aborted instead of waiting. See xyOps's [shared capacity and queue limits](https://docs.xyops.io/limits#shared-capacity-key). Use the same priority on all three Events unless you deliberately want to change queue order, and monitor jobs that fail or exhaust the queue.
|
|
179
|
+
|
|
180
|
+
Before enabling any of the three Events, export their new definitions from the checkout root:
|
|
181
|
+
|
|
182
|
+
```sh
|
|
183
|
+
cd /srv/xyops-automation
|
|
184
|
+
/usr/local/sbin/xyops-git-sync create
|
|
185
|
+
```
|
|
186
|
+
|
|
187
|
+
The `create` mode uses `sync setup --new` and its `--down_cmd` to commit and push those files. Review that commit and confirm the tree is complete. Then enable the three Events and test with disposable definitions before allowing normal Git pushes to trigger up-delete.
|
|
188
|
+
|
|
189
|
+
## 5. Trigger upsync from accepted Git pushes
|
|
190
|
+
|
|
191
|
+
Choose the Git trigger that matches your repository hosting. Both options launch the **same** Git Event after a push has been accepted. Do not enable both for one repository.
|
|
192
|
+
|
|
193
|
+
### Self-hosted Git: a `post-receive` hook
|
|
194
|
+
|
|
195
|
+
On a self-hosted **bare remote repository**, save the following as `hooks/post-receive` and make it executable. This hook runs on the Git server after refs have been updated, not in each contributor's clone or the sync worker's checkout. The Git server needs xyCLI credentials that permit launching the Git Event, but it does not need the sync checkout.
|
|
196
|
+
|
|
197
|
+
```sh
|
|
198
|
+
#!/bin/sh
|
|
199
|
+
set -eu
|
|
200
|
+
|
|
201
|
+
deploy=false
|
|
202
|
+
while read -r old_oid new_oid ref; do
|
|
203
|
+
[ "$ref" = refs/heads/main ] || continue
|
|
204
|
+
git cat-file -e "$new_oid^{commit}" 2>/dev/null || continue
|
|
205
|
+
|
|
206
|
+
# A commit generated by the down/create helper is already from xyOps.
|
|
207
|
+
# Do not turn its push into another authoritative up-delete run.
|
|
208
|
+
case "$(git log -1 --format=%s "$new_oid")" in
|
|
209
|
+
'[xyops-sync]'*) continue ;;
|
|
210
|
+
esac
|
|
211
|
+
deploy=true
|
|
212
|
+
done
|
|
213
|
+
|
|
214
|
+
[ "$deploy" = true ] || exit 0
|
|
215
|
+
/usr/local/bin/xy run SYNC_UP_EVENT_ID
|
|
216
|
+
```
|
|
217
|
+
|
|
218
|
+
Replace `SYNC_UP_EVENT_ID` with the xyOps Git Event's exact ID. Git's [`post-receive` hook](https://git-scm.com/docs/githooks#post-receive) runs after a push succeeds; a later hook or job failure cannot undo that push. Monitor the launched job and retry it after fixing any problem.
|
|
219
|
+
|
|
220
|
+
### GitHub: a `push` webhook
|
|
221
|
+
|
|
222
|
+
If GitHub hosts the repository, use its **push** webhook to notify your existing webhook receiver. In GitHub, open **Settings > Webhooks > Add webhook**, set its payload URL to that receiver, choose `application/json`, set a secret, and select only the **Pushes** event. The receiver should:
|
|
223
|
+
|
|
224
|
+
1. Verify `X-Hub-Signature-256` against the raw request body, and accept only the intended repository and `refs/heads/main`.
|
|
225
|
+
2. Ignore GitHub's initial `ping`, deleted branches, and commits whose head message starts with `[xyops-sync]`.
|
|
226
|
+
3. Deduplicate by `X-GitHub-Delivery`, then launch `SYNC_UP_EVENT_ID` through xyOps's `run_event` API using an API Key with `run_jobs` access.
|
|
227
|
+
4. Acknowledge only after recording the delivery for processing, and monitor the resulting xyOps job. Retry a failed job or redeliver a missed webhook after reviewing the Git and xyOps state.
|
|
228
|
+
|
|
229
|
+
The API request made by the verified receiver is a JSON `POST` to `/api/app/run_event/v1` with an `X-API-Key` header and body `{"id":"SYNC_UP_EVENT_ID"}`. The Event runs on the sync worker, so the receiver does not need the checkout. Do not put an xyOps API Key in the GitHub payload URL. A direct xyOps Magic Link can start an Event, but it does not verify GitHub's webhook signature; use a receiver that does that verification for this workflow. See GitHub's [webhook setup](https://docs.github.com/en/webhooks/using-webhooks/creating-webhooks), [signature validation](https://docs.github.com/en/webhooks/using-webhooks/validating-webhook-deliveries), and [delivery guidance](https://docs.github.com/en/webhooks/using-webhooks/best-practices-for-using-webhooks).
|
|
230
|
+
|
|
231
|
+
## 6. Test each direction before enabling automatic deletion
|
|
232
|
+
|
|
233
|
+
Run these tests with disposable Events and Plugins, not production definitions:
|
|
234
|
+
|
|
235
|
+
1. Create an Event or Plugin in xyOps. Confirm that the **create** Activity Event exports its XYPDF, commits it, and pushes it to Git.
|
|
236
|
+
2. Edit that definition in the xyOps UI. Confirm that the **down** Activity Event downloads the change and commits it.
|
|
237
|
+
3. Delete it in xyOps. Confirm that the **down** Event removes the XYPDF and owned neighbor files and pushes the deletion.
|
|
238
|
+
4. Edit an existing definition in Git and push. Confirm that the accepted-push trigger runs **up** and updates xyOps.
|
|
239
|
+
5. Delete a different existing definition in Git and push. Confirm that **up** deletes only that definition in xyOps.
|
|
240
|
+
|
|
241
|
+
Before the first live delete, preview each direction against the complete checkout and inspect every proposed deletion:
|
|
242
|
+
|
|
243
|
+
```sh
|
|
244
|
+
cd /srv/xyops-automation
|
|
245
|
+
xy sync . --up events,plugins --down false --delete events,plugins --dry
|
|
246
|
+
xy sync . --up false --down events,plugins --delete events,plugins --dry
|
|
247
|
+
```
|
|
248
|
+
|
|
249
|
+
Remember that a dry run previews the inventory **at that moment**. It cannot establish what a later Git push or xyOps activity means. Keep recoverable Git history and xyOps backups.
|
|
250
|
+
|
|
251
|
+
## Ordering and recovery
|
|
252
|
+
|
|
253
|
+
Authoritative triggers tell each pass which side should win. They do not make a full-inventory sweep transactional or remove all races. For example, an Event newly created in xyOps but not yet exported looks missing to an up-delete run; a new Git XYPDF that has not been created in xyOps looks missing to a down-delete run. This tutorial permits **creation only in xyOps**, so contributors should edit or delete existing XYPDF sources, not add new IDs in Git.
|
|
254
|
+
|
|
255
|
+
Keep all three jobs on the same worker and checkout, serialize the entire Git plus xyCLI sequence with their shared capacity key, and process pending xyOps creations before allowing a Git up-delete pass to treat the inventory as complete. The queue prevents overlap, but it cannot guarantee that independently delivered Git and xyOps triggers arrive in causal order. If changes on both sides occur together, pause automatic deletion, inspect the Git diff and both inventories, resolve the intended order, and rerun the appropriate job. Do not assume a successful webhook acknowledgment means the xyOps job succeeded.
|
|
256
|
+
|
|
257
|
+
If a job or push fails, leave the checkout and xyOps as they are until you have inspected them. A failed Git push can leave a local commit that needs to be pushed or reconciled before another `git pull --ff-only`. A failed Activity launch or webhook delivery needs a deliberate retry; there is no cron pass to pick it up later. GitHub [does not automatically redeliver failed webhook deliveries](https://docs.github.com/en/webhooks/using-webhooks/handling-failed-webhook-deliveries).
|
|
258
|
+
|
|
259
|
+
For all sync options and deletion safeguards, see the [Filesystem Sync Guide](sync.md#delete-mode). For a scheduled, modification-time-based workflow without automatic two-way deletion, see the [Two-Way Git Sync Tutorial](two-way-git-tutorial.md).
|
|
@@ -108,7 +108,9 @@ This considers both directions for every local Event and workflow source:
|
|
|
108
108
|
- A newer xyOps definition is downloaded to its existing local file.
|
|
109
109
|
- Identical definitions are left alone.
|
|
110
110
|
|
|
111
|
-
The `--delete false` switch is intentional.
|
|
111
|
+
The `--delete false` switch is intentional. A single command with both directions does not propagate deletions. Keep deletion disabled in this scheduled command, and follow the [coordinated deletion steps](#delete-a-definition-from-both-sides) when retiring an item. Separate one-way [up and down delete modes](sync.md#delete-mode) are available for workflows that can coordinate their triggers and inventories.
|
|
112
|
+
|
|
113
|
+
If you want deletion in both directions without a recurring schedule, use the separate [event-driven two-way delete tutorial](two-way-delete-tutorial.md).
|
|
112
114
|
|
|
113
115
|
Review the displayed diffs carefully. If the wrong side would win, check the clocks and file modification times before continuing.
|
|
114
116
|
|
package/lib/sync-buckets.js
CHANGED
|
@@ -87,6 +87,25 @@ module.exports = {
|
|
|
87
87
|
return { paths, data, data_stat: stat, files };
|
|
88
88
|
},
|
|
89
89
|
|
|
90
|
+
getSyncBucketLocalDeletePlan(item) {
|
|
91
|
+
// A deleted Bucket owns its sibling data.json and uploaded files. Only
|
|
92
|
+
// remove that directory when it has the exact structure produced by
|
|
93
|
+
// setup; an unexpected entry could belong to someone else.
|
|
94
|
+
var local = this.readSyncBucketLocal(item);
|
|
95
|
+
var dir_stat = fs.lstatSync(local.paths.dir);
|
|
96
|
+
if (!dir_stat.isDirectory() || dir_stat.isSymbolicLink()) {
|
|
97
|
+
throw new Error("Invalid bucket content directory: " + local.paths.dir);
|
|
98
|
+
}
|
|
99
|
+
var entries = fs.readdirSync(local.paths.dir).sort();
|
|
100
|
+
if (entries.length !== 2 || entries[0] !== 'data.json' || entries[1] !== 'files') {
|
|
101
|
+
throw new Error("Unexpected file in bucket content directory: " + local.paths.dir);
|
|
102
|
+
}
|
|
103
|
+
return {
|
|
104
|
+
files: [...local.files.values()].map( entry => entry.path ).concat(local.paths.data),
|
|
105
|
+
dirs: [local.paths.files, local.paths.dir]
|
|
106
|
+
};
|
|
107
|
+
},
|
|
108
|
+
|
|
90
109
|
stampSyncBucketFile(path, epoch) {
|
|
91
110
|
// Keep the access time intact. Server times are Unix seconds; Node accepts
|
|
92
111
|
// the same unit for utimes(), including fractional seconds if provided.
|
|
@@ -187,6 +206,7 @@ module.exports = {
|
|
|
187
206
|
var down = sconfig.down && sconfig.down.includes('buckets');
|
|
188
207
|
var delete_files = sconfig.delete && sconfig.delete.includes('buckets');
|
|
189
208
|
var deletes = [];
|
|
209
|
+
var local_deletes = [];
|
|
190
210
|
|
|
191
211
|
for (const entry of prepared) {
|
|
192
212
|
var id = entry.item.data.id;
|
|
@@ -233,6 +253,7 @@ module.exports = {
|
|
|
233
253
|
var target = manifest.get(name);
|
|
234
254
|
if (!target) {
|
|
235
255
|
if (up) uploads.push({ name, source });
|
|
256
|
+
else if (delete_files) local_deletes.push({ id, title, name, path: source.path });
|
|
236
257
|
continue;
|
|
237
258
|
}
|
|
238
259
|
var local_time = source.stat.mtimeMs / 1000;
|
|
@@ -294,7 +315,7 @@ module.exports = {
|
|
|
294
315
|
|
|
295
316
|
// Deletions are the final content phase. Any earlier warning or error
|
|
296
317
|
// suppresses all bucket file deletes, just like whole-object deletion.
|
|
297
|
-
if (deletes.length && (this.errors.length || this.warnings.length)) {
|
|
318
|
+
if ((deletes.length || local_deletes.length) && (this.errors.length || this.warnings.length)) {
|
|
298
319
|
this.logSyncWarning("Bucket file delete pass skipped because sync encountered warnings or errors.");
|
|
299
320
|
return;
|
|
300
321
|
}
|
|
@@ -311,5 +332,18 @@ module.exports = {
|
|
|
311
332
|
return;
|
|
312
333
|
}
|
|
313
334
|
}
|
|
335
|
+
for (const entry of local_deletes) {
|
|
336
|
+
this.printSyncBucketHeading('🗑️', this.dry ? 'Would delete local bucket file' : 'Deleting local bucket file', entry.title, entry.id, true);
|
|
337
|
+
println( " " + cli.emoji('📎') + " " + cli.chalk.bold.yellow(entry.name) );
|
|
338
|
+
if (this.dry) continue;
|
|
339
|
+
try {
|
|
340
|
+
fs.unlinkSync(entry.path);
|
|
341
|
+
this.downSynced = true;
|
|
342
|
+
}
|
|
343
|
+
catch (err) {
|
|
344
|
+
this.logSyncError("Local bucket file deletion failed: " + err);
|
|
345
|
+
return;
|
|
346
|
+
}
|
|
347
|
+
}
|
|
314
348
|
}
|
|
315
349
|
};
|
package/lib/sync.js
CHANGED
|
@@ -76,10 +76,10 @@ module.exports = {
|
|
|
76
76
|
if (sconfig.delete && !Tools.includesAll(supported_lists, sconfig.delete)) this.die("Invalid sync.delete list: " + sconfig.delete.join(', '));
|
|
77
77
|
if (sconfig.delete && !sconfig.delete.length) sconfig.delete = false;
|
|
78
78
|
|
|
79
|
-
// Deletion
|
|
80
|
-
//
|
|
81
|
-
if (sconfig.delete && (
|
|
82
|
-
this.die("Delete mode requires
|
|
79
|
+
// Deletion follows the sole enabled direction. Validate settings inherited
|
|
80
|
+
// from config as well as command-line args before scanning or writing.
|
|
81
|
+
if (sconfig.delete && (!!sconfig.up === !!sconfig.down)) {
|
|
82
|
+
this.die("Delete mode requires exactly one sync direction: --up or --down.");
|
|
83
83
|
}
|
|
84
84
|
|
|
85
85
|
// Bucket content needs the replacement API and data timestamps introduced
|
|
@@ -142,6 +142,7 @@ module.exports = {
|
|
|
142
142
|
|
|
143
143
|
// load all xypdf files
|
|
144
144
|
var items = [];
|
|
145
|
+
var local_delete_items = [];
|
|
145
146
|
var seen_sources = new Map();
|
|
146
147
|
|
|
147
148
|
files.filter( file => !!file.path.match(/\.json$/i) && !this.isSyncBucketExtraFile(file.path, bucket_extra_dirs) ).forEach( file => {
|
|
@@ -286,13 +287,25 @@ module.exports = {
|
|
|
286
287
|
// include neighbor metadata for downsync
|
|
287
288
|
if (!item.neighbors) item.neighbors = {};
|
|
288
289
|
item.neighbors[ prop_path ] = neighbor.path;
|
|
290
|
+
if (!item.delete_neighbors) item.delete_neighbors = [];
|
|
291
|
+
item.delete_neighbors.push(neighbor.path);
|
|
289
292
|
} ); // foreach neighbor
|
|
290
293
|
|
|
291
294
|
// do not include item if neighbor loading or validation failed
|
|
292
295
|
if (neighbor_error) return;
|
|
293
296
|
|
|
297
|
+
item.list = type_def.list;
|
|
298
|
+
item.file = file.path;
|
|
294
299
|
var xy_item = Tools.findObject(list, { id: item.data.id });
|
|
295
300
|
if (!xy_item) {
|
|
301
|
+
// In a down-only delete run, xyOps owns this definition type.
|
|
302
|
+
// Queue its local files for the final destructive pass, after every
|
|
303
|
+
// source and neighbor has passed the normal scan validation.
|
|
304
|
+
if (sconfig.delete && sconfig.down && sconfig.delete.includes(item.list)) {
|
|
305
|
+
local_delete_items.push(item);
|
|
306
|
+
return;
|
|
307
|
+
}
|
|
308
|
+
|
|
296
309
|
// An opted-in Git sync can briefly retain a source after its xyOps
|
|
297
310
|
// definition is deleted. Skip only that source and keep scanning.
|
|
298
311
|
if (ignore_missing) {
|
|
@@ -307,12 +320,6 @@ module.exports = {
|
|
|
307
320
|
// include effective mtime in item for two-way sync
|
|
308
321
|
item.mtime = file.mtime;
|
|
309
322
|
|
|
310
|
-
// include list name for convenience below
|
|
311
|
-
item.list = type_def.list;
|
|
312
|
-
|
|
313
|
-
// include file for downsync
|
|
314
|
-
item.file = file.path;
|
|
315
|
-
|
|
316
323
|
// add to sync list
|
|
317
324
|
items.push(item);
|
|
318
325
|
} ); // foreach json file
|
|
@@ -413,7 +420,12 @@ module.exports = {
|
|
|
413
420
|
// now handle deletes
|
|
414
421
|
if (!sconfig.delete) return await this.completeSync();
|
|
415
422
|
|
|
416
|
-
|
|
423
|
+
if (sconfig.down) {
|
|
424
|
+
this.deleteSyncLocalItems(local_delete_items);
|
|
425
|
+
return await this.completeSync();
|
|
426
|
+
}
|
|
427
|
+
|
|
428
|
+
// Up-only deletion removes remote definitions absent from the local tree.
|
|
417
429
|
var list_data_types = [];
|
|
418
430
|
for (var id in this.config.ui.data_types) {
|
|
419
431
|
var type_def = this.config.ui.data_types[id];
|
|
@@ -450,6 +462,95 @@ module.exports = {
|
|
|
450
462
|
await this.completeSync();
|
|
451
463
|
},
|
|
452
464
|
|
|
465
|
+
deleteSyncLocalItems(items) {
|
|
466
|
+
// Validate every selected local deletion before removing any files. The
|
|
467
|
+
// normal source scan has already checked XYPDF identity and neighboring
|
|
468
|
+
// property paths; recheck the source and file types in case a checkout or
|
|
469
|
+
// another process changed the tree while this sync was running.
|
|
470
|
+
var plans = [];
|
|
471
|
+
for (const item of items) {
|
|
472
|
+
// Match the protection used by upward deletion for bundled or
|
|
473
|
+
// Marketplace definitions. A local source may retain these markers
|
|
474
|
+
// even after its remote record has disappeared.
|
|
475
|
+
if (Object.hasOwn(item.data, 'stock') || Object.hasOwn(item.data, 'marketplace')) continue;
|
|
476
|
+
|
|
477
|
+
var bucket_plan = null;
|
|
478
|
+
try {
|
|
479
|
+
if (item.type === 'bucket') bucket_plan = this.getSyncBucketLocalDeletePlan(item);
|
|
480
|
+
}
|
|
481
|
+
catch (err) {
|
|
482
|
+
this.logSyncError("Local delete preflight failed: " + err.message);
|
|
483
|
+
return;
|
|
484
|
+
}
|
|
485
|
+
var targets = [...new Set(item.delete_neighbors || []), ...(bucket_plan ? bucket_plan.files : []), item.file];
|
|
486
|
+
var dirs = bucket_plan ? bucket_plan.dirs : [];
|
|
487
|
+
try {
|
|
488
|
+
for (const file of targets) {
|
|
489
|
+
var stat = fs.lstatSync(file);
|
|
490
|
+
if (!stat.isFile() || stat.isSymbolicLink()) throw new Error("Local deletion target is not a regular file: " + file);
|
|
491
|
+
}
|
|
492
|
+
for (const dir of dirs) {
|
|
493
|
+
var stat = fs.lstatSync(dir);
|
|
494
|
+
if (!stat.isDirectory() || stat.isSymbolicLink()) throw new Error("Local deletion target is not a regular directory: " + dir);
|
|
495
|
+
}
|
|
496
|
+
var source = Tools.parseJSON(fs.readFileSync(item.file, 'utf8'));
|
|
497
|
+
var source_item = source && Array.isArray(source.items) && (source.items.length === 1) ? source.items[0] : null;
|
|
498
|
+
if (!source || (source.type !== 'xypdf') || (source.version !== '1.0') || !source_item ||
|
|
499
|
+
(source_item.type !== item.type) || !source_item.data || (source_item.data.id !== item.data.id)) {
|
|
500
|
+
throw new Error("Local XYPDF source changed during sync: " + item.file);
|
|
501
|
+
}
|
|
502
|
+
}
|
|
503
|
+
catch (err) {
|
|
504
|
+
this.logSyncError("Local delete preflight failed: " + err.message);
|
|
505
|
+
return;
|
|
506
|
+
}
|
|
507
|
+
plans.push({ item, targets, dirs });
|
|
508
|
+
}
|
|
509
|
+
|
|
510
|
+
for (const plan of plans) {
|
|
511
|
+
println( "\n " + cli.emoji('🗑️') + ' ' + red.bold(this.dry ? "Would delete local " : "Deleting local ") +
|
|
512
|
+
red.bold(plan.item.type) + ": " + red.bold(plan.item.data.title || plan.item.data.id) +
|
|
513
|
+
gray(" (" + plan.item.data.id + ")") );
|
|
514
|
+
// Keep the XYPDF JSON source last so a partial filesystem failure
|
|
515
|
+
// leaves the identity available for a later retry.
|
|
516
|
+
for (const file of plan.targets.slice(0, -1)) {
|
|
517
|
+
println(" " + file);
|
|
518
|
+
if (this.dry) continue;
|
|
519
|
+
try {
|
|
520
|
+
fs.unlinkSync(file);
|
|
521
|
+
this.downSynced = true;
|
|
522
|
+
}
|
|
523
|
+
catch (err) {
|
|
524
|
+
this.logSyncError("Failed to delete local file: " + file + ": " + err.message);
|
|
525
|
+
return;
|
|
526
|
+
}
|
|
527
|
+
}
|
|
528
|
+
for (const dir of plan.dirs) {
|
|
529
|
+
println(" " + dir + Path.sep);
|
|
530
|
+
if (this.dry) continue;
|
|
531
|
+
try {
|
|
532
|
+
fs.rmdirSync(dir);
|
|
533
|
+
this.downSynced = true;
|
|
534
|
+
}
|
|
535
|
+
catch (err) {
|
|
536
|
+
this.logSyncError("Failed to delete local directory: " + dir + ": " + err.message);
|
|
537
|
+
return;
|
|
538
|
+
}
|
|
539
|
+
}
|
|
540
|
+
var source_file = plan.targets[plan.targets.length - 1];
|
|
541
|
+
println(" " + source_file);
|
|
542
|
+
if (this.dry) continue;
|
|
543
|
+
try {
|
|
544
|
+
fs.unlinkSync(source_file);
|
|
545
|
+
this.downSynced = true;
|
|
546
|
+
}
|
|
547
|
+
catch (err) {
|
|
548
|
+
this.logSyncError("Failed to delete local file: " + source_file + ": " + err.message);
|
|
549
|
+
return;
|
|
550
|
+
}
|
|
551
|
+
}
|
|
552
|
+
},
|
|
553
|
+
|
|
453
554
|
getSyncLockFile() {
|
|
454
555
|
// Allow a fixed path for service installations. Otherwise, serialize all
|
|
455
556
|
// local syncs targeting the exact same configured xyOps base URL. Local
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@pixlcore/xycli",
|
|
3
|
-
"version": "1.0
|
|
3
|
+
"version": "1.1.0",
|
|
4
4
|
"description": "A complete command-line interface for the xyOps workflow automation system.",
|
|
5
5
|
"author": "Joseph Huckaby <jhuckaby@pixlcore.com>",
|
|
6
6
|
"homepage": "https://github.com/pixlcore/xycli",
|
package/test/README.md
CHANGED
|
@@ -66,10 +66,10 @@ node --test test/config.test.js test/sync-buckets-unit.test.js test/sync-unit.te
|
|
|
66
66
|
| `secrets` | Vault metadata, assignments, redacted previews, full field replacement, confirmed decryption, deletion, validation, help |
|
|
67
67
|
| `servers` | Active and recently offline lists, local filters, installer generation, historical search, live and historical Server details, charts, processes, connections, pagination, validation, help |
|
|
68
68
|
| `system` | Admin dashboard, conductors, connected users, export, maintenance, optimization, rate resets, diagnostics, broadcasts, help |
|
|
69
|
-
| `sync` | Setup export, isolated filesystem discovery, external property files, dry-run, upsync, downsync, composite global-state governance, no-op detection, and
|
|
70
|
-
| `sync-buckets` | Disposable Bucket setup, JSON replacement, file upload and download,
|
|
69
|
+
| `sync` | Setup export, isolated filesystem discovery, external property files, dry-run, upsync, downsync, composite global-state governance, no-op detection, up-only deletion, and down-only Event and Plugin file deletion |
|
|
70
|
+
| `sync-buckets` | Disposable Bucket setup, JSON replacement, file upload and download, timestamp alignment, and down-only local file and whole-Bucket deletion on xyOps 1.1.2 or later |
|
|
71
71
|
| `sync-buckets-unit` | Offline Bucket version gating, content setup, upload and download, timestamp alignment, and pre-delete safety |
|
|
72
|
-
| `sync-unit` | Offline setup extension fallbacks, new-only exports and setup completion commands, PID locking and stale recovery,
|
|
72
|
+
| `sync-unit` | Offline setup extension fallbacks, new-only exports and setup completion commands, PID locking and stale recovery, one-way deletion guards, pre-delete failure blocking, stock/Marketplace deletion protection, inherited settings and CLI overrides, warning/error exit status, duplicates, quiet/dry runs, completion failures, and notification delivery after warnings or errors |
|
|
73
73
|
| `tags` | CRUD, metadata search, sparse future-proof updates, JSON input, validation, pagination, help |
|
|
74
74
|
| `tickets` | Search, CRUD, number resolution, comments, attachment uploads/downloads, Events, related Job lookup, sparse updates, validation, help |
|
|
75
75
|
| `toast` | Warning-box word wrapping and terminal-width limits |
|
|
@@ -127,6 +127,37 @@ test('downsync replaces changed files and preserves the server timestamp', async
|
|
|
127
127
|
assert.equal(fs.statSync(env.paths.data).mtimeMs / 1000, 150);
|
|
128
128
|
});
|
|
129
129
|
|
|
130
|
+
test('down-only bucket deletion removes local files absent from the remote manifest', async t => {
|
|
131
|
+
const keep = { filename: 'keep.txt', path: 'files/bucket/bucket_unit/key/keep.txt', size: 4, date: 80 };
|
|
132
|
+
const env = fixture(t, { localData: { keep: true }, remoteData: { keep: true }, remoteFiles: [keep] });
|
|
133
|
+
const keep_path = Path.join(env.paths.files, 'keep.txt');
|
|
134
|
+
const stale_path = Path.join(env.paths.files, 'stale.txt');
|
|
135
|
+
fs.writeFileSync(keep_path, 'keep');
|
|
136
|
+
fs.utimesSync(keep_path, 80, 80);
|
|
137
|
+
fs.writeFileSync(stale_path, 'stale');
|
|
138
|
+
|
|
139
|
+
await env.app.syncBucketContents([env.item], { down: ['buckets'], delete: ['buckets'] });
|
|
140
|
+
assert.equal(env.app.errors.length, 0);
|
|
141
|
+
assert.equal(fs.existsSync(keep_path), true);
|
|
142
|
+
assert.equal(fs.existsSync(stale_path), false);
|
|
143
|
+
assert.equal(env.app.downSynced, true);
|
|
144
|
+
assert.deepEqual(env.calls, []);
|
|
145
|
+
});
|
|
146
|
+
|
|
147
|
+
test('down-only bucket deletion previews files and skips them after a failed download', async t => {
|
|
148
|
+
const remote_file = { filename: 'remote.txt', path: 'files/bucket/bucket_unit/key/remote.txt', size: 6, date: 80 };
|
|
149
|
+
for (const dry of [true, false]) {
|
|
150
|
+
const env = fixture(t, { localData: { keep: true }, remoteData: { keep: true },
|
|
151
|
+
remoteFiles: [remote_file], dry });
|
|
152
|
+
const stale_path = Path.join(env.paths.files, 'stale.txt');
|
|
153
|
+
fs.writeFileSync(stale_path, 'stale');
|
|
154
|
+
await env.app.syncBucketContents([env.item], { down: ['buckets'], delete: ['buckets'] });
|
|
155
|
+
assert.equal(fs.existsSync(stale_path), true);
|
|
156
|
+
if (dry) assert.equal(env.app.errors.length, 0);
|
|
157
|
+
else assert.match(env.app.errors[0], /Missing remote bytes/);
|
|
158
|
+
}
|
|
159
|
+
});
|
|
160
|
+
|
|
130
161
|
test('two-way sync chooses the newer data and file copies independently', async t => {
|
|
131
162
|
const old = { filename: 'remote.txt', path: 'files/bucket/bucket_unit/key/remote.txt', size: 6, date: 140 };
|
|
132
163
|
const env = fixture(t, {
|
|
@@ -115,6 +115,42 @@ test('bucket filesystem sync', async t => {
|
|
|
115
115
|
assert.equal(Math.floor(fs.statSync(file).mtimeMs / 1000), remoteFile.date);
|
|
116
116
|
assert.equal(Math.floor(fs.statSync(dataFile).mtimeMs / 1000), bucket.meta.mod);
|
|
117
117
|
});
|
|
118
|
+
|
|
119
|
+
await check('down-only delete removes a local Bucket file absent from xyOps', async () => {
|
|
120
|
+
const content = Path.join(activeRoot, 'buckets', slug);
|
|
121
|
+
const extra = Path.join(content, 'files', 'LocalOnly.txt');
|
|
122
|
+
fs.writeFileSync(extra, 'only in the local sync tree');
|
|
123
|
+
const before = await call('getBucket', { id });
|
|
124
|
+
|
|
125
|
+
const preview = xy(['sync', activeRoot, '--up', 'false', '--down', 'buckets', '--delete', 'buckets', '--dry']);
|
|
126
|
+
assert.match(preview, /Would delete local bucket file/);
|
|
127
|
+
assert.ok(fs.existsSync(extra), 'Dry run preserves the local file');
|
|
128
|
+
|
|
129
|
+
const output = xy(['sync', activeRoot, '--up', 'false', '--down', 'buckets', '--delete', 'buckets']);
|
|
130
|
+
assert.match(output, /Deleting local bucket file/);
|
|
131
|
+
assert.ok(!fs.existsSync(extra), 'Missing remote file was removed locally');
|
|
132
|
+
assert.ok(fs.existsSync(Path.join(content, 'files', 'seed.json')), 'Existing remote file remains');
|
|
133
|
+
assert.deepEqual((await call('getBucket', { id })).files, before.files, 'Remote Bucket files are unchanged');
|
|
134
|
+
});
|
|
135
|
+
|
|
136
|
+
await check('down-only delete removes a deleted Bucket and its local content', async () => {
|
|
137
|
+
const source = Path.join(activeRoot, 'buckets', slug + '.json');
|
|
138
|
+
const content = Path.join(activeRoot, 'buckets', slug);
|
|
139
|
+
await call('deleteBucket', { id });
|
|
140
|
+
created = false;
|
|
141
|
+
assert.ok(fs.existsSync(source), 'Local source remains before down-delete');
|
|
142
|
+
assert.ok(fs.existsSync(content), 'Local content remains before down-delete');
|
|
143
|
+
|
|
144
|
+
const preview = xy(['sync', activeRoot, '--up', 'false', '--down', 'buckets', '--delete', 'buckets', '--dry']);
|
|
145
|
+
assert.match(preview, /Would delete local bucket/);
|
|
146
|
+
assert.ok(fs.existsSync(source), 'Dry run preserves the Bucket source');
|
|
147
|
+
assert.ok(fs.existsSync(content), 'Dry run preserves Bucket content');
|
|
148
|
+
|
|
149
|
+
const output = xy(['sync', activeRoot, '--up', 'false', '--down', 'buckets', '--delete', 'buckets']);
|
|
150
|
+
assert.match(output, /Deleting local bucket/);
|
|
151
|
+
assert.ok(!fs.existsSync(source), 'Deleted Bucket source was removed');
|
|
152
|
+
assert.ok(!fs.existsSync(content), 'Deleted Bucket content was removed');
|
|
153
|
+
});
|
|
118
154
|
}
|
|
119
155
|
finally {
|
|
120
156
|
// Ordinary sync writes a shared management map. Restore it exactly, then
|
package/test/sync-unit.test.js
CHANGED
|
@@ -15,26 +15,30 @@ const script = `
|
|
|
15
15
|
const options = JSON.parse(process.argv[1]);
|
|
16
16
|
cli.args.quiet = options.quiet !== false;
|
|
17
17
|
const sync = require('./lib/sync.js');
|
|
18
|
+
const bucketSync = require('./lib/sync-buckets.js');
|
|
18
19
|
if (options.setupCwd) process.chdir(options.setupCwd);
|
|
19
20
|
const calls = [];
|
|
20
21
|
const requests = [];
|
|
21
22
|
const apiOptions = [];
|
|
22
23
|
const app = {
|
|
23
24
|
...sync,
|
|
25
|
+
...bucketSync,
|
|
24
26
|
args: options.args,
|
|
25
27
|
config: {
|
|
26
28
|
base_url: 'https://fixture.invalid',
|
|
27
29
|
sync: options.config || {},
|
|
28
30
|
ui: {
|
|
29
|
-
list_list: ['categories', 'events', 'plugins'].map(id => ({ id })),
|
|
30
|
-
data_types: { category: { list: 'categories' }, event: { list: 'events' }, plugin: { list: 'plugins' } }
|
|
31
|
+
list_list: (options.includeBuckets ? ['buckets', 'categories', 'events', 'plugins'] : ['categories', 'events', 'plugins']).map(id => ({ id })),
|
|
32
|
+
data_types: { bucket: { list: 'buckets' }, category: { list: 'categories' }, event: { list: 'events' }, plugin: { list: 'plugins' } }
|
|
31
33
|
}
|
|
32
34
|
},
|
|
33
|
-
state: {},
|
|
35
|
+
state: options.state || {},
|
|
36
|
+
buckets: options.buckets || [],
|
|
34
37
|
categories: options.categories || [],
|
|
35
38
|
events: options.events || [],
|
|
36
39
|
plugins: options.plugins || [],
|
|
37
40
|
version: 'test',
|
|
41
|
+
xyopsVersion: '1.1.2',
|
|
38
42
|
dry: !!options.args.dry,
|
|
39
43
|
color() { return cli.chalk.white; },
|
|
40
44
|
markdown(text) { return text; },
|
|
@@ -95,6 +99,15 @@ function runSync(t, options) {
|
|
|
95
99
|
items: [{ type: source.type, data: source.data }]
|
|
96
100
|
});
|
|
97
101
|
fs.writeFileSync(file, contents);
|
|
102
|
+
if (source.bucketFiles) {
|
|
103
|
+
var bucket_dir = file.replace(/\.json$/i, '');
|
|
104
|
+
fs.mkdirSync(Path.join(bucket_dir, 'files'), { recursive: true });
|
|
105
|
+
fs.writeFileSync(Path.join(bucket_dir, 'data.json'), JSON.stringify(source.bucketData || {}));
|
|
106
|
+
for (const [name, contents] of Object.entries(source.bucketFiles)) {
|
|
107
|
+
fs.writeFileSync(Path.join(bucket_dir, 'files', name), contents);
|
|
108
|
+
}
|
|
109
|
+
if (source.bucketExtra) fs.writeFileSync(Path.join(bucket_dir, source.bucketExtra), 'keep');
|
|
110
|
+
}
|
|
98
111
|
});
|
|
99
112
|
}
|
|
100
113
|
if (options.missingDir) args.other = [Path.join(dir, 'missing')];
|
|
@@ -189,16 +202,16 @@ test('sync update guard rejects malformed confirmation values', async () => {
|
|
|
189
202
|
);
|
|
190
203
|
});
|
|
191
204
|
|
|
192
|
-
test('sync rejects deletion
|
|
205
|
+
test('sync rejects deletion without exactly one direction before writes', t => {
|
|
193
206
|
for (const args of [
|
|
194
|
-
{ down: 'categories', delete: 'categories' },
|
|
195
207
|
{ up: 'categories', down: 'categories', delete: 'categories', dry: true },
|
|
196
208
|
{ up: 'categories', down: 'events', delete: 'categories' },
|
|
197
|
-
{ up: [], delete: 'categories' }
|
|
209
|
+
{ up: [], delete: 'categories' },
|
|
210
|
+
{ down: [], delete: 'categories' }
|
|
198
211
|
]) {
|
|
199
212
|
const result = runSync(t, { args });
|
|
200
213
|
assert.equal(result.status, 1);
|
|
201
|
-
assert.match(result.stderr, /Delete mode requires
|
|
214
|
+
assert.match(result.stderr, /Delete mode requires exactly one sync direction/);
|
|
202
215
|
assert.deepEqual(result.report.calls, []);
|
|
203
216
|
}
|
|
204
217
|
});
|
|
@@ -218,6 +231,101 @@ test('sync validates inherited deletion and direction settings after CLI overrid
|
|
|
218
231
|
assert.equal(emptyDown.status, 0);
|
|
219
232
|
});
|
|
220
233
|
|
|
234
|
+
test('down-only delete removes one missing local source and its neighbors', t => {
|
|
235
|
+
const result = runSync(t, {
|
|
236
|
+
categories: [{ id: 'keep', title: 'Keep', notes: 'Same' }],
|
|
237
|
+
events: [{ id: 'missing', title: 'Same ID on another type' }],
|
|
238
|
+
existingSources: [
|
|
239
|
+
{ path: 'categories/Stale.json', type: 'category', data: { id: 'missing', title: 'Stale', notes: '(External)' } },
|
|
240
|
+
{ path: 'categories/Stale-notes.txt', contents: 'Old note\n' },
|
|
241
|
+
{ path: 'categories/Keep.json', type: 'category', data: { id: 'keep', title: 'Keep', notes: 'Same' } },
|
|
242
|
+
{ path: 'events/Other.json', type: 'event', data: { id: 'missing', title: 'Same ID on another type' } }
|
|
243
|
+
],
|
|
244
|
+
args: { down: 'categories', delete: 'categories', down_cmd: getSetupMarkerCommand('completed.txt') }
|
|
245
|
+
});
|
|
246
|
+
assert.equal(result.status, 0, result.stderr);
|
|
247
|
+
assert.equal(fs.existsSync(Path.join(result.dir, 'categories/Stale.json')), false);
|
|
248
|
+
assert.equal(fs.existsSync(Path.join(result.dir, 'categories/Stale-notes.txt')), false);
|
|
249
|
+
assert.equal(fs.existsSync(Path.join(result.dir, 'categories/Keep.json')), true);
|
|
250
|
+
assert.equal(fs.existsSync(Path.join(result.dir, 'events/Other.json')), true);
|
|
251
|
+
assert.equal(fs.readFileSync(Path.join(result.dir, 'completed.txt'), 'utf8'), fs.realpathSync(result.dir));
|
|
252
|
+
assert.deepEqual(result.report.requests.filter(req => req.method.startsWith('delete_')), []);
|
|
253
|
+
});
|
|
254
|
+
|
|
255
|
+
test('down-only delete previews local files without changing them or running down_cmd', t => {
|
|
256
|
+
const result = runSync(t, {
|
|
257
|
+
quiet: false,
|
|
258
|
+
existingSources: [
|
|
259
|
+
{ path: 'categories/Stale.json', type: 'category', data: { id: 'missing', title: 'Stale' } }
|
|
260
|
+
],
|
|
261
|
+
args: { down: 'categories', delete: 'categories', dry: true, down_cmd: getSetupMarkerCommand('completed.txt') }
|
|
262
|
+
});
|
|
263
|
+
assert.equal(result.status, 0, result.stderr);
|
|
264
|
+
assert.match(result.stdout, /Would delete local/);
|
|
265
|
+
assert.equal(fs.existsSync(Path.join(result.dir, 'categories/Stale.json')), true);
|
|
266
|
+
assert.equal(fs.existsSync(Path.join(result.dir, 'completed.txt')), false);
|
|
267
|
+
});
|
|
268
|
+
|
|
269
|
+
test('down-only delete removes an orphan Bucket and all its local content', t => {
|
|
270
|
+
const result = runSync(t, {
|
|
271
|
+
includeBuckets: true,
|
|
272
|
+
existingSources: [
|
|
273
|
+
{ path: 'buckets/Stale.json', type: 'bucket', data: { id: 'missing', title: 'Stale' },
|
|
274
|
+
bucketData: { old: true }, bucketFiles: { 'old.txt': 'contents' } }
|
|
275
|
+
],
|
|
276
|
+
args: { down: 'buckets', delete: 'buckets' }
|
|
277
|
+
});
|
|
278
|
+
assert.equal(result.status, 0, result.stderr);
|
|
279
|
+
assert.equal(fs.existsSync(Path.join(result.dir, 'buckets/Stale.json')), false);
|
|
280
|
+
assert.equal(fs.existsSync(Path.join(result.dir, 'buckets/Stale')), false);
|
|
281
|
+
});
|
|
282
|
+
|
|
283
|
+
test('down-only delete preflights all Bucket content before removing any source', t => {
|
|
284
|
+
const result = runSync(t, {
|
|
285
|
+
includeBuckets: true,
|
|
286
|
+
existingSources: [
|
|
287
|
+
{ path: 'buckets/First.json', type: 'bucket', data: { id: 'first', title: 'First' }, bucketFiles: {} },
|
|
288
|
+
{ path: 'buckets/Second.json', type: 'bucket', data: { id: 'second', title: 'Second' },
|
|
289
|
+
bucketFiles: {}, bucketExtra: 'untracked.txt' }
|
|
290
|
+
],
|
|
291
|
+
args: { down: 'buckets', delete: 'buckets' }
|
|
292
|
+
});
|
|
293
|
+
assert.equal(result.status, 1);
|
|
294
|
+
assert.match(result.report.errors[0], /Local delete preflight failed/);
|
|
295
|
+
assert.equal(fs.existsSync(Path.join(result.dir, 'buckets/First.json')), true);
|
|
296
|
+
assert.equal(fs.existsSync(Path.join(result.dir, 'buckets/First')), true);
|
|
297
|
+
assert.equal(fs.existsSync(Path.join(result.dir, 'buckets/Second.json')), true);
|
|
298
|
+
});
|
|
299
|
+
|
|
300
|
+
test('down-only delete protects local stock and Marketplace definitions', t => {
|
|
301
|
+
const result = runSync(t, {
|
|
302
|
+
existingSources: [
|
|
303
|
+
{ path: 'categories/Stock.json', type: 'category', data: { id: 'stock', title: 'Stock', stock: false } },
|
|
304
|
+
{ path: 'categories/Marketplace.json', type: 'category', data: { id: 'marketplace', title: 'Marketplace', marketplace: null } },
|
|
305
|
+
{ path: 'categories/Custom.json', type: 'category', data: { id: 'custom', title: 'Custom' } }
|
|
306
|
+
],
|
|
307
|
+
args: { down: 'categories', delete: 'categories' }
|
|
308
|
+
});
|
|
309
|
+
assert.equal(result.status, 0, result.stderr);
|
|
310
|
+
assert.equal(fs.existsSync(Path.join(result.dir, 'categories/Stock.json')), true);
|
|
311
|
+
assert.equal(fs.existsSync(Path.join(result.dir, 'categories/Marketplace.json')), true);
|
|
312
|
+
assert.equal(fs.existsSync(Path.join(result.dir, 'categories/Custom.json')), false);
|
|
313
|
+
});
|
|
314
|
+
|
|
315
|
+
test('down-only delete skips local removals after a sync-state error', t => {
|
|
316
|
+
const result = runSync(t, {
|
|
317
|
+
state: { sync: { 'category-old': true } },
|
|
318
|
+
failAPI: 'update_global_state',
|
|
319
|
+
existingSources: [
|
|
320
|
+
{ path: 'categories/Stale.json', type: 'category', data: { id: 'stale', title: 'Stale' } }
|
|
321
|
+
],
|
|
322
|
+
args: { down: 'categories', delete: 'categories' }
|
|
323
|
+
});
|
|
324
|
+
assert.equal(result.status, 1);
|
|
325
|
+
assert.match(result.report.warnings[0], /Delete pass skipped/);
|
|
326
|
+
assert.equal(fs.existsSync(Path.join(result.dir, 'categories/Stale.json')), true);
|
|
327
|
+
});
|
|
328
|
+
|
|
221
329
|
test('sync errors exit nonzero even in quiet mode or a dry run', t => {
|
|
222
330
|
for (const dry of [false, true]) {
|
|
223
331
|
const result = runSync(t, { missingDir: true, args: { up: 'categories', dry, quiet: true } });
|
package/test/sync.test.js
CHANGED
|
@@ -510,6 +510,48 @@ test('sync', async t => {
|
|
|
510
510
|
assert.ok(fixtures.category && fixtures.plugin && fixtures.event, 'Resources outside the missing fixture remain');
|
|
511
511
|
assert.deepEqual(fixtures.syncState, deleteSyncState, 'Up-only delete mode tracks the remaining Category sources');
|
|
512
512
|
});
|
|
513
|
+
|
|
514
|
+
await check('remove disposable remote Event and Plugin for down-only deletion', async () => {
|
|
515
|
+
// The local sources and external property files remain in the isolated
|
|
516
|
+
// sync tree. Delete only these two server fixtures so down-delete has
|
|
517
|
+
// exactly two local candidates and no unrelated remote mutations.
|
|
518
|
+
await call('deleteEvent', { id: eventID });
|
|
519
|
+
await call('deletePlugin', { id: pluginID });
|
|
520
|
+
const fixtures = await getFixtures();
|
|
521
|
+
assert.equal(fixtures.event, undefined);
|
|
522
|
+
assert.equal(fixtures.plugin, undefined);
|
|
523
|
+
assert.ok(fixtures.category, 'Unrelated Category remains in xyOps');
|
|
524
|
+
for (const file of [eventFile, eventScriptFile, pluginFile, pluginScriptFile, categoryFile]) {
|
|
525
|
+
assert.ok(fs.existsSync(file), 'Local fixture remains before down-delete: ' + file);
|
|
526
|
+
}
|
|
527
|
+
});
|
|
528
|
+
|
|
529
|
+
await check('dry down-only delete preserves local Event and Plugin files', () => {
|
|
530
|
+
const output = xy([
|
|
531
|
+
'sync', syncRoot, '--up', 'false', '--down', 'events,plugins', '--delete', 'events,plugins', '--dry'
|
|
532
|
+
], { cwd: temp });
|
|
533
|
+
assert.match(output, /Would delete local event/);
|
|
534
|
+
assert.match(output, /Would delete local plugin/);
|
|
535
|
+
for (const file of [eventFile, eventScriptFile, pluginFile, pluginScriptFile, categoryFile]) {
|
|
536
|
+
assert.ok(fs.existsSync(file), 'Dry run preserves local fixture: ' + file);
|
|
537
|
+
}
|
|
538
|
+
});
|
|
539
|
+
|
|
540
|
+
await check('down-only delete removes only missing Event and Plugin files', async () => {
|
|
541
|
+
const output = xy([
|
|
542
|
+
'sync', syncRoot, '--up', 'false', '--down', 'events,plugins', '--delete', 'events,plugins'
|
|
543
|
+
], { cwd: temp });
|
|
544
|
+
assert.match(output, /Deleting local event/);
|
|
545
|
+
assert.match(output, /Deleting local plugin/);
|
|
546
|
+
for (const file of [eventFile, eventScriptFile, pluginFile, pluginScriptFile]) {
|
|
547
|
+
assert.ok(!fs.existsSync(file), 'Down-delete removes local fixture: ' + file);
|
|
548
|
+
}
|
|
549
|
+
assert.ok(fs.existsSync(categoryFile), 'Unrelated local Category remains');
|
|
550
|
+
const fixtures = await getFixtures();
|
|
551
|
+
assert.ok(fixtures.category, 'Down-delete preserves unrelated remote Category');
|
|
552
|
+
assert.equal(fixtures.event, undefined);
|
|
553
|
+
assert.equal(fixtures.plugin, undefined);
|
|
554
|
+
});
|
|
513
555
|
}
|
|
514
556
|
finally {
|
|
515
557
|
const cleanupErrors = [];
|
package/test/tickets.test.js
CHANGED
|
@@ -23,6 +23,7 @@ test('tickets', async t => {
|
|
|
23
23
|
const body = '# Test Ticket\n\n- **Bold item** with `code`\n';
|
|
24
24
|
let ticketCreated = false;
|
|
25
25
|
let tagCreated = false;
|
|
26
|
+
let commentAuthor = '';
|
|
26
27
|
|
|
27
28
|
fs.writeFileSync(bodyFile, body);
|
|
28
29
|
fs.writeFileSync(firstFile, 'First Ticket attachment.\n');
|
|
@@ -110,6 +111,8 @@ test('tickets', async t => {
|
|
|
110
111
|
const saved = row.changes.filter( change => change.type == 'comment' );
|
|
111
112
|
assert.equal(saved.length, 1);
|
|
112
113
|
assert.equal(saved[0].body, comment);
|
|
114
|
+
// The API key can belong to any user, so use the author saved by xyOps.
|
|
115
|
+
commentAuthor = saved[0].username || '(Unknown)';
|
|
113
116
|
assert.equal(row.status, 'draft');
|
|
114
117
|
});
|
|
115
118
|
|
|
@@ -159,7 +162,7 @@ test('tickets', async t => {
|
|
|
159
162
|
'TICKET #' + num, 'Number', '#' + num, 'Ticket ID', id, subject,
|
|
160
163
|
'TICKET BODY', 'Test Ticket', '• Bold item with code',
|
|
161
164
|
'TICKET FILES', Path.basename(firstFile), Path.basename(secondFile), 'Download a file',
|
|
162
|
-
'Comment by
|
|
165
|
+
'Comment by ' + commentAuthor, 'Investigation update', '• Checked the logs'
|
|
163
166
|
]) assert.ok(out.includes(text), text);
|
|
164
167
|
if (event) {
|
|
165
168
|
assert.match(out, /TICKET EVENTS/);
|