@spica-devkit/sync 0.20.7 → 0.20.8

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/index.js CHANGED
@@ -36,6 +36,85 @@ var ChangeKind;
36
36
  })(ChangeKind || (ChangeKind = {}));
37
37
  const DEFAULT_CONCURRENCY = 10;
38
38
 
39
+ const REQUEST_METHODS = ["get", "delete", "head", "post", "patch", "put"];
40
+ const limitedClients = new WeakMap();
41
+ function limitConcurrency(http, limit = DEFAULT_CONCURRENCY, signal) {
42
+ const existing = limitedClients.get(http);
43
+ if (existing && !signal)
44
+ return http;
45
+ const { semaphore, target } = existing ?? {
46
+ semaphore: new Semaphore(normalizeLimit(limit)),
47
+ target: http
48
+ };
49
+ const limited = { ...target };
50
+ for (const method of REQUEST_METHODS) {
51
+ limited[method] = (...args) => semaphore.run(() => {
52
+ if (signal?.aborted)
53
+ throw signal.reason;
54
+ return target[method](...args);
55
+ });
56
+ }
57
+ limitedClients.set(limited, { semaphore, target });
58
+ return limited;
59
+ }
60
+ async function mapWithConcurrency(items, limit, fn) {
61
+ const results = new Array(items.length);
62
+ let next = 0;
63
+ let failure;
64
+ const worker = async () => {
65
+ while (!failure && next < items.length) {
66
+ const index = next++;
67
+ try {
68
+ results[index] = await fn(items[index], index);
69
+ }
70
+ catch (error) {
71
+ if (!failure)
72
+ failure = { error };
73
+ }
74
+ }
75
+ };
76
+ const workerCount = Math.min(normalizeLimit(limit), items.length);
77
+ await Promise.all(Array.from({ length: workerCount }, worker));
78
+ if (failure)
79
+ throw failure.error;
80
+ return results;
81
+ }
82
+ function normalizeLimit(limit) {
83
+ return Number.isFinite(limit) ? Math.max(1, Math.floor(limit)) : DEFAULT_CONCURRENCY;
84
+ }
85
+ class Semaphore {
86
+ constructor(limit) {
87
+ this.limit = limit;
88
+ this.active = 0;
89
+ this.waiting = [];
90
+ }
91
+ async run(task) {
92
+ await this.acquire();
93
+ try {
94
+ return await task();
95
+ }
96
+ finally {
97
+ this.release();
98
+ }
99
+ }
100
+ acquire() {
101
+ if (this.active < this.limit) {
102
+ this.active++;
103
+ return Promise.resolve();
104
+ }
105
+ return new Promise(resolve => this.waiting.push(resolve));
106
+ }
107
+ release() {
108
+ const next = this.waiting.shift();
109
+ if (next) {
110
+ next();
111
+ }
112
+ else {
113
+ this.active--;
114
+ }
115
+ }
116
+ }
117
+
39
118
  const noopProgress = {
40
119
  update() { },
41
120
  succeed() { },
@@ -12735,13 +12814,14 @@ function createTwoFilesPatch(oldFileName, newFileName, oldStr, newStr, oldHeader
12735
12814
  return ret.join('\n') + '\n';
12736
12815
  }
12737
12816
 
12738
- async function buildPlan(modules, http, rootDir, options = {}) {
12817
+ async function buildPlan(modules, unlimitedHttp, rootDir, options = {}) {
12739
12818
  const { detailed = false, detectRenames = true, reporter = silentReporter, concurrency = DEFAULT_CONCURRENCY, source } = options;
12819
+ const http = limitConcurrency(unlimitedHttp, concurrency);
12740
12820
  const modulePlans = [];
12741
12821
  for (const mod of modules) {
12742
12822
  const [locals, remotes] = await Promise.all([
12743
12823
  reporter.task(`Reading local ${mod.displayName} files`, () => mod.readLocal(rootDir, source)),
12744
- reporter.task(`Fetching remote ${mod.displayName}`, () => mod.readRemote(http, { concurrency }))
12824
+ reporter.task(`Fetching remote ${mod.displayName}`, () => mod.readRemote(http))
12745
12825
  ]);
12746
12826
  const localBySlug = new Map(locals.map(l => [l.slug, l]));
12747
12827
  const remoteBySlug = new Map(remotes.map(r => [r.slug, r]));
@@ -12820,29 +12900,30 @@ async function buildPlan(modules, http, rootDir, options = {}) {
12820
12900
  }
12821
12901
  return { modules: modulePlans };
12822
12902
  }
12823
- async function applyPlan(plan, http, opts = {}) {
12903
+ async function applyPlan(plan, unlimitedHttp, opts = {}) {
12824
12904
  const concurrency = opts.concurrency ?? DEFAULT_CONCURRENCY;
12825
12905
  const reporter = opts.reporter ?? silentReporter;
12826
12906
  const errors = [];
12907
+ const abortController = new AbortController();
12908
+ const http = limitConcurrency(unlimitedHttp, concurrency, abortController.signal);
12909
+ let abortError;
12827
12910
  const handleErr = (label, e) => {
12828
- const msg = formatError(e);
12829
- const full = `${label}: ${msg}`;
12830
- if (opts.abortOnError)
12831
- throw new Error(full);
12911
+ if (abortError)
12912
+ throw abortError;
12913
+ const full = `${label}: ${formatError(e)}`;
12914
+ if (opts.abortOnError) {
12915
+ abortError = new Error(full);
12916
+ abortController.abort(abortError);
12917
+ throw abortError;
12918
+ }
12832
12919
  reporter.warn(full);
12833
12920
  errors.push(full);
12834
12921
  };
12835
12922
  for (const mp of plan.modules) {
12836
12923
  const { module: mod, creates, updates, deletes } = mp;
12837
- if (deletes.length) {
12838
- await runConcurrently(deletes.map(e => ({ slug: e.slug, run: () => mod.delete(http, e.remote.id) })), concurrency, `Deleting ${mod.displayName}`, handleErr, reporter);
12839
- }
12840
- if (updates.length) {
12841
- await runConcurrently(updates.map(e => ({ slug: e.slug, run: () => mod.update(http, e.local, e.remote.id) })), concurrency, `Updating ${mod.displayName}`, handleErr, reporter);
12842
- }
12843
- if (creates.length) {
12844
- await runConcurrently(creates.map(e => ({ slug: e.slug, run: () => mod.create(http, e.local) })), concurrency, `Creating ${mod.displayName}`, handleErr, reporter);
12845
- }
12924
+ await runAllAtOnce(deletes.map(e => ({ slug: e.slug, run: () => mod.delete(http, e.remote.id) })), `Deleting ${mod.displayName}`, handleErr, reporter);
12925
+ await runAllAtOnce(updates.map(e => ({ slug: e.slug, run: () => mod.update(http, e.local, e.remote.id) })), `Updating ${mod.displayName}`, handleErr, reporter);
12926
+ await runAllAtOnce(creates.map(e => ({ slug: e.slug, run: () => mod.create(http, e.local) })), `Creating ${mod.displayName}`, handleErr, reporter);
12846
12927
  }
12847
12928
  return { errors };
12848
12929
  }
@@ -12869,6 +12950,9 @@ function renderSchemaDetail(localData, remoteData, ignoredFields) {
12869
12950
  const remoteYaml = YAML.stringify(omit(remoteData, ignoredFields));
12870
12951
  return { schema: buildUnifiedDiff(remoteYaml, localYaml, "schema.yaml") };
12871
12952
  }
12953
+ function runAllAtOnce(tasks, label, onError, reporter) {
12954
+ return runConcurrently(tasks, tasks.length, label, onError, reporter);
12955
+ }
12872
12956
  async function runConcurrently(tasks, concurrency, label, onError, reporter) {
12873
12957
  if (!tasks.length)
12874
12958
  return;
@@ -12876,21 +12960,13 @@ async function runConcurrently(tasks, concurrency, label, onError, reporter) {
12876
12960
  let done = 0;
12877
12961
  const progress = reporter.progress(label, total);
12878
12962
  try {
12879
- const limit = Math.max(1, Math.min(total, concurrency));
12880
- let batch = [];
12881
- for (let i = 0; i < tasks.length; i++) {
12882
- batch.push(tasks[i]);
12883
- if (batch.length === limit || i === tasks.length - 1) {
12884
- await Promise.all(batch.map(task => task
12885
- .run()
12886
- .catch((e) => onError(`${label}: ${task.slug}`, e))
12887
- .finally(() => {
12888
- done++;
12889
- progress.update(task.slug, done, total);
12890
- })));
12891
- batch = [];
12892
- }
12893
- }
12963
+ await mapWithConcurrency(tasks, concurrency, task => task
12964
+ .run()
12965
+ .catch((e) => onError(`${label}: ${task.slug}`, e))
12966
+ .finally(() => {
12967
+ done++;
12968
+ progress.update(task.slug, done, total);
12969
+ }));
12894
12970
  progress.succeed();
12895
12971
  }
12896
12972
  catch (e) {
@@ -15747,14 +15823,6 @@ function normalizeSchema(schema) {
15747
15823
  }
15748
15824
  return normalized;
15749
15825
  }
15750
- async function mapInBatches(items, size, fn) {
15751
- const limit = Math.max(1, size);
15752
- const results = [];
15753
- for (let i = 0; i < items.length; i += limit) {
15754
- results.push(...(await Promise.all(items.slice(i, i + limit).map(fn))));
15755
- }
15756
- return results;
15757
- }
15758
15826
  function isNotFoundError(error) {
15759
15827
  if (!error || typeof error !== "object")
15760
15828
  return false;
@@ -15818,10 +15886,10 @@ const functionModule = {
15818
15886
  }
15819
15887
  return results;
15820
15888
  },
15821
- async readRemote(http, options = {}) {
15889
+ async readRemote(http) {
15822
15890
  const res = await http.get("function");
15823
15891
  const fns = unwrapList(res);
15824
- return mapInBatches(fns, options.concurrency ?? DEFAULT_CONCURRENCY, async (fn) => {
15892
+ return Promise.all(fns.map(async (fn) => {
15825
15893
  const id = fn._id;
15826
15894
  const [indexRes, depsRes] = await Promise.all([
15827
15895
  http.get(`function/${id}/index`).catch(emptyIfMissing({ index: "" })),
@@ -15844,7 +15912,7 @@ const functionModule = {
15844
15912
  dependencies
15845
15913
  }
15846
15914
  };
15847
- });
15915
+ }));
15848
15916
  },
15849
15917
  async create(http, local) {
15850
15918
  const id = await this.createSchema(http, local.data.schema);