@spica-devkit/sync 0.20.7 → 0.20.8
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/index.js +110 -42
- package/index.js.map +1 -1
- package/index.mjs +110 -42
- package/index.mjs.map +1 -1
- package/package.json +1 -1
package/index.js
CHANGED
|
@@ -36,6 +36,85 @@ var ChangeKind;
|
|
|
36
36
|
})(ChangeKind || (ChangeKind = {}));
|
|
37
37
|
const DEFAULT_CONCURRENCY = 10;
|
|
38
38
|
|
|
39
|
+
const REQUEST_METHODS = ["get", "delete", "head", "post", "patch", "put"];
|
|
40
|
+
const limitedClients = new WeakMap();
|
|
41
|
+
function limitConcurrency(http, limit = DEFAULT_CONCURRENCY, signal) {
|
|
42
|
+
const existing = limitedClients.get(http);
|
|
43
|
+
if (existing && !signal)
|
|
44
|
+
return http;
|
|
45
|
+
const { semaphore, target } = existing ?? {
|
|
46
|
+
semaphore: new Semaphore(normalizeLimit(limit)),
|
|
47
|
+
target: http
|
|
48
|
+
};
|
|
49
|
+
const limited = { ...target };
|
|
50
|
+
for (const method of REQUEST_METHODS) {
|
|
51
|
+
limited[method] = (...args) => semaphore.run(() => {
|
|
52
|
+
if (signal?.aborted)
|
|
53
|
+
throw signal.reason;
|
|
54
|
+
return target[method](...args);
|
|
55
|
+
});
|
|
56
|
+
}
|
|
57
|
+
limitedClients.set(limited, { semaphore, target });
|
|
58
|
+
return limited;
|
|
59
|
+
}
|
|
60
|
+
async function mapWithConcurrency(items, limit, fn) {
|
|
61
|
+
const results = new Array(items.length);
|
|
62
|
+
let next = 0;
|
|
63
|
+
let failure;
|
|
64
|
+
const worker = async () => {
|
|
65
|
+
while (!failure && next < items.length) {
|
|
66
|
+
const index = next++;
|
|
67
|
+
try {
|
|
68
|
+
results[index] = await fn(items[index], index);
|
|
69
|
+
}
|
|
70
|
+
catch (error) {
|
|
71
|
+
if (!failure)
|
|
72
|
+
failure = { error };
|
|
73
|
+
}
|
|
74
|
+
}
|
|
75
|
+
};
|
|
76
|
+
const workerCount = Math.min(normalizeLimit(limit), items.length);
|
|
77
|
+
await Promise.all(Array.from({ length: workerCount }, worker));
|
|
78
|
+
if (failure)
|
|
79
|
+
throw failure.error;
|
|
80
|
+
return results;
|
|
81
|
+
}
|
|
82
|
+
function normalizeLimit(limit) {
|
|
83
|
+
return Number.isFinite(limit) ? Math.max(1, Math.floor(limit)) : DEFAULT_CONCURRENCY;
|
|
84
|
+
}
|
|
85
|
+
class Semaphore {
|
|
86
|
+
constructor(limit) {
|
|
87
|
+
this.limit = limit;
|
|
88
|
+
this.active = 0;
|
|
89
|
+
this.waiting = [];
|
|
90
|
+
}
|
|
91
|
+
async run(task) {
|
|
92
|
+
await this.acquire();
|
|
93
|
+
try {
|
|
94
|
+
return await task();
|
|
95
|
+
}
|
|
96
|
+
finally {
|
|
97
|
+
this.release();
|
|
98
|
+
}
|
|
99
|
+
}
|
|
100
|
+
acquire() {
|
|
101
|
+
if (this.active < this.limit) {
|
|
102
|
+
this.active++;
|
|
103
|
+
return Promise.resolve();
|
|
104
|
+
}
|
|
105
|
+
return new Promise(resolve => this.waiting.push(resolve));
|
|
106
|
+
}
|
|
107
|
+
release() {
|
|
108
|
+
const next = this.waiting.shift();
|
|
109
|
+
if (next) {
|
|
110
|
+
next();
|
|
111
|
+
}
|
|
112
|
+
else {
|
|
113
|
+
this.active--;
|
|
114
|
+
}
|
|
115
|
+
}
|
|
116
|
+
}
|
|
117
|
+
|
|
39
118
|
const noopProgress = {
|
|
40
119
|
update() { },
|
|
41
120
|
succeed() { },
|
|
@@ -12735,13 +12814,14 @@ function createTwoFilesPatch(oldFileName, newFileName, oldStr, newStr, oldHeader
|
|
|
12735
12814
|
return ret.join('\n') + '\n';
|
|
12736
12815
|
}
|
|
12737
12816
|
|
|
12738
|
-
async function buildPlan(modules,
|
|
12817
|
+
async function buildPlan(modules, unlimitedHttp, rootDir, options = {}) {
|
|
12739
12818
|
const { detailed = false, detectRenames = true, reporter = silentReporter, concurrency = DEFAULT_CONCURRENCY, source } = options;
|
|
12819
|
+
const http = limitConcurrency(unlimitedHttp, concurrency);
|
|
12740
12820
|
const modulePlans = [];
|
|
12741
12821
|
for (const mod of modules) {
|
|
12742
12822
|
const [locals, remotes] = await Promise.all([
|
|
12743
12823
|
reporter.task(`Reading local ${mod.displayName} files`, () => mod.readLocal(rootDir, source)),
|
|
12744
|
-
reporter.task(`Fetching remote ${mod.displayName}`, () => mod.readRemote(http
|
|
12824
|
+
reporter.task(`Fetching remote ${mod.displayName}`, () => mod.readRemote(http))
|
|
12745
12825
|
]);
|
|
12746
12826
|
const localBySlug = new Map(locals.map(l => [l.slug, l]));
|
|
12747
12827
|
const remoteBySlug = new Map(remotes.map(r => [r.slug, r]));
|
|
@@ -12820,29 +12900,30 @@ async function buildPlan(modules, http, rootDir, options = {}) {
|
|
|
12820
12900
|
}
|
|
12821
12901
|
return { modules: modulePlans };
|
|
12822
12902
|
}
|
|
12823
|
-
async function applyPlan(plan,
|
|
12903
|
+
async function applyPlan(plan, unlimitedHttp, opts = {}) {
|
|
12824
12904
|
const concurrency = opts.concurrency ?? DEFAULT_CONCURRENCY;
|
|
12825
12905
|
const reporter = opts.reporter ?? silentReporter;
|
|
12826
12906
|
const errors = [];
|
|
12907
|
+
const abortController = new AbortController();
|
|
12908
|
+
const http = limitConcurrency(unlimitedHttp, concurrency, abortController.signal);
|
|
12909
|
+
let abortError;
|
|
12827
12910
|
const handleErr = (label, e) => {
|
|
12828
|
-
|
|
12829
|
-
|
|
12830
|
-
|
|
12831
|
-
|
|
12911
|
+
if (abortError)
|
|
12912
|
+
throw abortError;
|
|
12913
|
+
const full = `${label}: ${formatError(e)}`;
|
|
12914
|
+
if (opts.abortOnError) {
|
|
12915
|
+
abortError = new Error(full);
|
|
12916
|
+
abortController.abort(abortError);
|
|
12917
|
+
throw abortError;
|
|
12918
|
+
}
|
|
12832
12919
|
reporter.warn(full);
|
|
12833
12920
|
errors.push(full);
|
|
12834
12921
|
};
|
|
12835
12922
|
for (const mp of plan.modules) {
|
|
12836
12923
|
const { module: mod, creates, updates, deletes } = mp;
|
|
12837
|
-
|
|
12838
|
-
|
|
12839
|
-
}
|
|
12840
|
-
if (updates.length) {
|
|
12841
|
-
await runConcurrently(updates.map(e => ({ slug: e.slug, run: () => mod.update(http, e.local, e.remote.id) })), concurrency, `Updating ${mod.displayName}`, handleErr, reporter);
|
|
12842
|
-
}
|
|
12843
|
-
if (creates.length) {
|
|
12844
|
-
await runConcurrently(creates.map(e => ({ slug: e.slug, run: () => mod.create(http, e.local) })), concurrency, `Creating ${mod.displayName}`, handleErr, reporter);
|
|
12845
|
-
}
|
|
12924
|
+
await runAllAtOnce(deletes.map(e => ({ slug: e.slug, run: () => mod.delete(http, e.remote.id) })), `Deleting ${mod.displayName}`, handleErr, reporter);
|
|
12925
|
+
await runAllAtOnce(updates.map(e => ({ slug: e.slug, run: () => mod.update(http, e.local, e.remote.id) })), `Updating ${mod.displayName}`, handleErr, reporter);
|
|
12926
|
+
await runAllAtOnce(creates.map(e => ({ slug: e.slug, run: () => mod.create(http, e.local) })), `Creating ${mod.displayName}`, handleErr, reporter);
|
|
12846
12927
|
}
|
|
12847
12928
|
return { errors };
|
|
12848
12929
|
}
|
|
@@ -12869,6 +12950,9 @@ function renderSchemaDetail(localData, remoteData, ignoredFields) {
|
|
|
12869
12950
|
const remoteYaml = YAML.stringify(omit(remoteData, ignoredFields));
|
|
12870
12951
|
return { schema: buildUnifiedDiff(remoteYaml, localYaml, "schema.yaml") };
|
|
12871
12952
|
}
|
|
12953
|
+
function runAllAtOnce(tasks, label, onError, reporter) {
|
|
12954
|
+
return runConcurrently(tasks, tasks.length, label, onError, reporter);
|
|
12955
|
+
}
|
|
12872
12956
|
async function runConcurrently(tasks, concurrency, label, onError, reporter) {
|
|
12873
12957
|
if (!tasks.length)
|
|
12874
12958
|
return;
|
|
@@ -12876,21 +12960,13 @@ async function runConcurrently(tasks, concurrency, label, onError, reporter) {
|
|
|
12876
12960
|
let done = 0;
|
|
12877
12961
|
const progress = reporter.progress(label, total);
|
|
12878
12962
|
try {
|
|
12879
|
-
|
|
12880
|
-
|
|
12881
|
-
|
|
12882
|
-
|
|
12883
|
-
|
|
12884
|
-
|
|
12885
|
-
|
|
12886
|
-
.catch((e) => onError(`${label}: ${task.slug}`, e))
|
|
12887
|
-
.finally(() => {
|
|
12888
|
-
done++;
|
|
12889
|
-
progress.update(task.slug, done, total);
|
|
12890
|
-
})));
|
|
12891
|
-
batch = [];
|
|
12892
|
-
}
|
|
12893
|
-
}
|
|
12963
|
+
await mapWithConcurrency(tasks, concurrency, task => task
|
|
12964
|
+
.run()
|
|
12965
|
+
.catch((e) => onError(`${label}: ${task.slug}`, e))
|
|
12966
|
+
.finally(() => {
|
|
12967
|
+
done++;
|
|
12968
|
+
progress.update(task.slug, done, total);
|
|
12969
|
+
}));
|
|
12894
12970
|
progress.succeed();
|
|
12895
12971
|
}
|
|
12896
12972
|
catch (e) {
|
|
@@ -15747,14 +15823,6 @@ function normalizeSchema(schema) {
|
|
|
15747
15823
|
}
|
|
15748
15824
|
return normalized;
|
|
15749
15825
|
}
|
|
15750
|
-
async function mapInBatches(items, size, fn) {
|
|
15751
|
-
const limit = Math.max(1, size);
|
|
15752
|
-
const results = [];
|
|
15753
|
-
for (let i = 0; i < items.length; i += limit) {
|
|
15754
|
-
results.push(...(await Promise.all(items.slice(i, i + limit).map(fn))));
|
|
15755
|
-
}
|
|
15756
|
-
return results;
|
|
15757
|
-
}
|
|
15758
15826
|
function isNotFoundError(error) {
|
|
15759
15827
|
if (!error || typeof error !== "object")
|
|
15760
15828
|
return false;
|
|
@@ -15818,10 +15886,10 @@ const functionModule = {
|
|
|
15818
15886
|
}
|
|
15819
15887
|
return results;
|
|
15820
15888
|
},
|
|
15821
|
-
async readRemote(http
|
|
15889
|
+
async readRemote(http) {
|
|
15822
15890
|
const res = await http.get("function");
|
|
15823
15891
|
const fns = unwrapList(res);
|
|
15824
|
-
return
|
|
15892
|
+
return Promise.all(fns.map(async (fn) => {
|
|
15825
15893
|
const id = fn._id;
|
|
15826
15894
|
const [indexRes, depsRes] = await Promise.all([
|
|
15827
15895
|
http.get(`function/${id}/index`).catch(emptyIfMissing({ index: "" })),
|
|
@@ -15844,7 +15912,7 @@ const functionModule = {
|
|
|
15844
15912
|
dependencies
|
|
15845
15913
|
}
|
|
15846
15914
|
};
|
|
15847
|
-
});
|
|
15915
|
+
}));
|
|
15848
15916
|
},
|
|
15849
15917
|
async create(http, local) {
|
|
15850
15918
|
const id = await this.createSchema(http, local.data.schema);
|