@spica-devkit/sync 0.20.6 → 0.20.8
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/index.js +110 -42
- package/index.js.map +1 -1
- package/index.mjs +110 -42
- package/index.mjs.map +1 -1
- package/package.json +1 -1
package/index.mjs
CHANGED
|
@@ -15,6 +15,85 @@ var ChangeKind;
|
|
|
15
15
|
})(ChangeKind || (ChangeKind = {}));
|
|
16
16
|
const DEFAULT_CONCURRENCY = 10;
|
|
17
17
|
|
|
18
|
+
const REQUEST_METHODS = ["get", "delete", "head", "post", "patch", "put"];
|
|
19
|
+
const limitedClients = new WeakMap();
|
|
20
|
+
function limitConcurrency(http, limit = DEFAULT_CONCURRENCY, signal) {
|
|
21
|
+
const existing = limitedClients.get(http);
|
|
22
|
+
if (existing && !signal)
|
|
23
|
+
return http;
|
|
24
|
+
const { semaphore, target } = existing ?? {
|
|
25
|
+
semaphore: new Semaphore(normalizeLimit(limit)),
|
|
26
|
+
target: http
|
|
27
|
+
};
|
|
28
|
+
const limited = { ...target };
|
|
29
|
+
for (const method of REQUEST_METHODS) {
|
|
30
|
+
limited[method] = (...args) => semaphore.run(() => {
|
|
31
|
+
if (signal?.aborted)
|
|
32
|
+
throw signal.reason;
|
|
33
|
+
return target[method](...args);
|
|
34
|
+
});
|
|
35
|
+
}
|
|
36
|
+
limitedClients.set(limited, { semaphore, target });
|
|
37
|
+
return limited;
|
|
38
|
+
}
|
|
39
|
+
async function mapWithConcurrency(items, limit, fn) {
|
|
40
|
+
const results = new Array(items.length);
|
|
41
|
+
let next = 0;
|
|
42
|
+
let failure;
|
|
43
|
+
const worker = async () => {
|
|
44
|
+
while (!failure && next < items.length) {
|
|
45
|
+
const index = next++;
|
|
46
|
+
try {
|
|
47
|
+
results[index] = await fn(items[index], index);
|
|
48
|
+
}
|
|
49
|
+
catch (error) {
|
|
50
|
+
if (!failure)
|
|
51
|
+
failure = { error };
|
|
52
|
+
}
|
|
53
|
+
}
|
|
54
|
+
};
|
|
55
|
+
const workerCount = Math.min(normalizeLimit(limit), items.length);
|
|
56
|
+
await Promise.all(Array.from({ length: workerCount }, worker));
|
|
57
|
+
if (failure)
|
|
58
|
+
throw failure.error;
|
|
59
|
+
return results;
|
|
60
|
+
}
|
|
61
|
+
function normalizeLimit(limit) {
|
|
62
|
+
return Number.isFinite(limit) ? Math.max(1, Math.floor(limit)) : DEFAULT_CONCURRENCY;
|
|
63
|
+
}
|
|
64
|
+
class Semaphore {
|
|
65
|
+
constructor(limit) {
|
|
66
|
+
this.limit = limit;
|
|
67
|
+
this.active = 0;
|
|
68
|
+
this.waiting = [];
|
|
69
|
+
}
|
|
70
|
+
async run(task) {
|
|
71
|
+
await this.acquire();
|
|
72
|
+
try {
|
|
73
|
+
return await task();
|
|
74
|
+
}
|
|
75
|
+
finally {
|
|
76
|
+
this.release();
|
|
77
|
+
}
|
|
78
|
+
}
|
|
79
|
+
acquire() {
|
|
80
|
+
if (this.active < this.limit) {
|
|
81
|
+
this.active++;
|
|
82
|
+
return Promise.resolve();
|
|
83
|
+
}
|
|
84
|
+
return new Promise(resolve => this.waiting.push(resolve));
|
|
85
|
+
}
|
|
86
|
+
release() {
|
|
87
|
+
const next = this.waiting.shift();
|
|
88
|
+
if (next) {
|
|
89
|
+
next();
|
|
90
|
+
}
|
|
91
|
+
else {
|
|
92
|
+
this.active--;
|
|
93
|
+
}
|
|
94
|
+
}
|
|
95
|
+
}
|
|
96
|
+
|
|
18
97
|
const noopProgress = {
|
|
19
98
|
update() { },
|
|
20
99
|
succeed() { },
|
|
@@ -12714,13 +12793,14 @@ function createTwoFilesPatch(oldFileName, newFileName, oldStr, newStr, oldHeader
|
|
|
12714
12793
|
return ret.join('\n') + '\n';
|
|
12715
12794
|
}
|
|
12716
12795
|
|
|
12717
|
-
async function buildPlan(modules,
|
|
12796
|
+
async function buildPlan(modules, unlimitedHttp, rootDir, options = {}) {
|
|
12718
12797
|
const { detailed = false, detectRenames = true, reporter = silentReporter, concurrency = DEFAULT_CONCURRENCY, source } = options;
|
|
12798
|
+
const http = limitConcurrency(unlimitedHttp, concurrency);
|
|
12719
12799
|
const modulePlans = [];
|
|
12720
12800
|
for (const mod of modules) {
|
|
12721
12801
|
const [locals, remotes] = await Promise.all([
|
|
12722
12802
|
reporter.task(`Reading local ${mod.displayName} files`, () => mod.readLocal(rootDir, source)),
|
|
12723
|
-
reporter.task(`Fetching remote ${mod.displayName}`, () => mod.readRemote(http
|
|
12803
|
+
reporter.task(`Fetching remote ${mod.displayName}`, () => mod.readRemote(http))
|
|
12724
12804
|
]);
|
|
12725
12805
|
const localBySlug = new Map(locals.map(l => [l.slug, l]));
|
|
12726
12806
|
const remoteBySlug = new Map(remotes.map(r => [r.slug, r]));
|
|
@@ -12799,29 +12879,30 @@ async function buildPlan(modules, http, rootDir, options = {}) {
|
|
|
12799
12879
|
}
|
|
12800
12880
|
return { modules: modulePlans };
|
|
12801
12881
|
}
|
|
12802
|
-
async function applyPlan(plan,
|
|
12882
|
+
async function applyPlan(plan, unlimitedHttp, opts = {}) {
|
|
12803
12883
|
const concurrency = opts.concurrency ?? DEFAULT_CONCURRENCY;
|
|
12804
12884
|
const reporter = opts.reporter ?? silentReporter;
|
|
12805
12885
|
const errors = [];
|
|
12886
|
+
const abortController = new AbortController();
|
|
12887
|
+
const http = limitConcurrency(unlimitedHttp, concurrency, abortController.signal);
|
|
12888
|
+
let abortError;
|
|
12806
12889
|
const handleErr = (label, e) => {
|
|
12807
|
-
|
|
12808
|
-
|
|
12809
|
-
|
|
12810
|
-
|
|
12890
|
+
if (abortError)
|
|
12891
|
+
throw abortError;
|
|
12892
|
+
const full = `${label}: ${formatError(e)}`;
|
|
12893
|
+
if (opts.abortOnError) {
|
|
12894
|
+
abortError = new Error(full);
|
|
12895
|
+
abortController.abort(abortError);
|
|
12896
|
+
throw abortError;
|
|
12897
|
+
}
|
|
12811
12898
|
reporter.warn(full);
|
|
12812
12899
|
errors.push(full);
|
|
12813
12900
|
};
|
|
12814
12901
|
for (const mp of plan.modules) {
|
|
12815
12902
|
const { module: mod, creates, updates, deletes } = mp;
|
|
12816
|
-
|
|
12817
|
-
|
|
12818
|
-
}
|
|
12819
|
-
if (updates.length) {
|
|
12820
|
-
await runConcurrently(updates.map(e => ({ slug: e.slug, run: () => mod.update(http, e.local, e.remote.id) })), concurrency, `Updating ${mod.displayName}`, handleErr, reporter);
|
|
12821
|
-
}
|
|
12822
|
-
if (creates.length) {
|
|
12823
|
-
await runConcurrently(creates.map(e => ({ slug: e.slug, run: () => mod.create(http, e.local) })), concurrency, `Creating ${mod.displayName}`, handleErr, reporter);
|
|
12824
|
-
}
|
|
12903
|
+
await runAllAtOnce(deletes.map(e => ({ slug: e.slug, run: () => mod.delete(http, e.remote.id) })), `Deleting ${mod.displayName}`, handleErr, reporter);
|
|
12904
|
+
await runAllAtOnce(updates.map(e => ({ slug: e.slug, run: () => mod.update(http, e.local, e.remote.id) })), `Updating ${mod.displayName}`, handleErr, reporter);
|
|
12905
|
+
await runAllAtOnce(creates.map(e => ({ slug: e.slug, run: () => mod.create(http, e.local) })), `Creating ${mod.displayName}`, handleErr, reporter);
|
|
12825
12906
|
}
|
|
12826
12907
|
return { errors };
|
|
12827
12908
|
}
|
|
@@ -12848,6 +12929,9 @@ function renderSchemaDetail(localData, remoteData, ignoredFields) {
|
|
|
12848
12929
|
const remoteYaml = YAML.stringify(omit(remoteData, ignoredFields));
|
|
12849
12930
|
return { schema: buildUnifiedDiff(remoteYaml, localYaml, "schema.yaml") };
|
|
12850
12931
|
}
|
|
12932
|
+
function runAllAtOnce(tasks, label, onError, reporter) {
|
|
12933
|
+
return runConcurrently(tasks, tasks.length, label, onError, reporter);
|
|
12934
|
+
}
|
|
12851
12935
|
async function runConcurrently(tasks, concurrency, label, onError, reporter) {
|
|
12852
12936
|
if (!tasks.length)
|
|
12853
12937
|
return;
|
|
@@ -12855,21 +12939,13 @@ async function runConcurrently(tasks, concurrency, label, onError, reporter) {
|
|
|
12855
12939
|
let done = 0;
|
|
12856
12940
|
const progress = reporter.progress(label, total);
|
|
12857
12941
|
try {
|
|
12858
|
-
|
|
12859
|
-
|
|
12860
|
-
|
|
12861
|
-
|
|
12862
|
-
|
|
12863
|
-
|
|
12864
|
-
|
|
12865
|
-
.catch((e) => onError(`${label}: ${task.slug}`, e))
|
|
12866
|
-
.finally(() => {
|
|
12867
|
-
done++;
|
|
12868
|
-
progress.update(task.slug, done, total);
|
|
12869
|
-
})));
|
|
12870
|
-
batch = [];
|
|
12871
|
-
}
|
|
12872
|
-
}
|
|
12942
|
+
await mapWithConcurrency(tasks, concurrency, task => task
|
|
12943
|
+
.run()
|
|
12944
|
+
.catch((e) => onError(`${label}: ${task.slug}`, e))
|
|
12945
|
+
.finally(() => {
|
|
12946
|
+
done++;
|
|
12947
|
+
progress.update(task.slug, done, total);
|
|
12948
|
+
}));
|
|
12873
12949
|
progress.succeed();
|
|
12874
12950
|
}
|
|
12875
12951
|
catch (e) {
|
|
@@ -15726,14 +15802,6 @@ function normalizeSchema(schema) {
|
|
|
15726
15802
|
}
|
|
15727
15803
|
return normalized;
|
|
15728
15804
|
}
|
|
15729
|
-
async function mapInBatches(items, size, fn) {
|
|
15730
|
-
const limit = Math.max(1, size);
|
|
15731
|
-
const results = [];
|
|
15732
|
-
for (let i = 0; i < items.length; i += limit) {
|
|
15733
|
-
results.push(...(await Promise.all(items.slice(i, i + limit).map(fn))));
|
|
15734
|
-
}
|
|
15735
|
-
return results;
|
|
15736
|
-
}
|
|
15737
15805
|
function isNotFoundError(error) {
|
|
15738
15806
|
if (!error || typeof error !== "object")
|
|
15739
15807
|
return false;
|
|
@@ -15797,10 +15865,10 @@ const functionModule = {
|
|
|
15797
15865
|
}
|
|
15798
15866
|
return results;
|
|
15799
15867
|
},
|
|
15800
|
-
async readRemote(http
|
|
15868
|
+
async readRemote(http) {
|
|
15801
15869
|
const res = await http.get("function");
|
|
15802
15870
|
const fns = unwrapList(res);
|
|
15803
|
-
return
|
|
15871
|
+
return Promise.all(fns.map(async (fn) => {
|
|
15804
15872
|
const id = fn._id;
|
|
15805
15873
|
const [indexRes, depsRes] = await Promise.all([
|
|
15806
15874
|
http.get(`function/${id}/index`).catch(emptyIfMissing({ index: "" })),
|
|
@@ -15823,7 +15891,7 @@ const functionModule = {
|
|
|
15823
15891
|
dependencies
|
|
15824
15892
|
}
|
|
15825
15893
|
};
|
|
15826
|
-
});
|
|
15894
|
+
}));
|
|
15827
15895
|
},
|
|
15828
15896
|
async create(http, local) {
|
|
15829
15897
|
const id = await this.createSchema(http, local.data.schema);
|