@spica-devkit/sync 0.20.6 → 0.20.8

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/index.mjs CHANGED
@@ -15,6 +15,85 @@ var ChangeKind;
15
15
  })(ChangeKind || (ChangeKind = {}));
16
16
  const DEFAULT_CONCURRENCY = 10;
17
17
 
18
+ const REQUEST_METHODS = ["get", "delete", "head", "post", "patch", "put"];
19
+ const limitedClients = new WeakMap();
20
+ function limitConcurrency(http, limit = DEFAULT_CONCURRENCY, signal) {
21
+ const existing = limitedClients.get(http);
22
+ if (existing && !signal)
23
+ return http;
24
+ const { semaphore, target } = existing ?? {
25
+ semaphore: new Semaphore(normalizeLimit(limit)),
26
+ target: http
27
+ };
28
+ const limited = { ...target };
29
+ for (const method of REQUEST_METHODS) {
30
+ limited[method] = (...args) => semaphore.run(() => {
31
+ if (signal?.aborted)
32
+ throw signal.reason;
33
+ return target[method](...args);
34
+ });
35
+ }
36
+ limitedClients.set(limited, { semaphore, target });
37
+ return limited;
38
+ }
39
+ async function mapWithConcurrency(items, limit, fn) {
40
+ const results = new Array(items.length);
41
+ let next = 0;
42
+ let failure;
43
+ const worker = async () => {
44
+ while (!failure && next < items.length) {
45
+ const index = next++;
46
+ try {
47
+ results[index] = await fn(items[index], index);
48
+ }
49
+ catch (error) {
50
+ if (!failure)
51
+ failure = { error };
52
+ }
53
+ }
54
+ };
55
+ const workerCount = Math.min(normalizeLimit(limit), items.length);
56
+ await Promise.all(Array.from({ length: workerCount }, worker));
57
+ if (failure)
58
+ throw failure.error;
59
+ return results;
60
+ }
61
+ function normalizeLimit(limit) {
62
+ return Number.isFinite(limit) ? Math.max(1, Math.floor(limit)) : DEFAULT_CONCURRENCY;
63
+ }
64
+ class Semaphore {
65
+ constructor(limit) {
66
+ this.limit = limit;
67
+ this.active = 0;
68
+ this.waiting = [];
69
+ }
70
+ async run(task) {
71
+ await this.acquire();
72
+ try {
73
+ return await task();
74
+ }
75
+ finally {
76
+ this.release();
77
+ }
78
+ }
79
+ acquire() {
80
+ if (this.active < this.limit) {
81
+ this.active++;
82
+ return Promise.resolve();
83
+ }
84
+ return new Promise(resolve => this.waiting.push(resolve));
85
+ }
86
+ release() {
87
+ const next = this.waiting.shift();
88
+ if (next) {
89
+ next();
90
+ }
91
+ else {
92
+ this.active--;
93
+ }
94
+ }
95
+ }
96
+
18
97
  const noopProgress = {
19
98
  update() { },
20
99
  succeed() { },
@@ -12714,13 +12793,14 @@ function createTwoFilesPatch(oldFileName, newFileName, oldStr, newStr, oldHeader
12714
12793
  return ret.join('\n') + '\n';
12715
12794
  }
12716
12795
 
12717
- async function buildPlan(modules, http, rootDir, options = {}) {
12796
+ async function buildPlan(modules, unlimitedHttp, rootDir, options = {}) {
12718
12797
  const { detailed = false, detectRenames = true, reporter = silentReporter, concurrency = DEFAULT_CONCURRENCY, source } = options;
12798
+ const http = limitConcurrency(unlimitedHttp, concurrency);
12719
12799
  const modulePlans = [];
12720
12800
  for (const mod of modules) {
12721
12801
  const [locals, remotes] = await Promise.all([
12722
12802
  reporter.task(`Reading local ${mod.displayName} files`, () => mod.readLocal(rootDir, source)),
12723
- reporter.task(`Fetching remote ${mod.displayName}`, () => mod.readRemote(http, { concurrency }))
12803
+ reporter.task(`Fetching remote ${mod.displayName}`, () => mod.readRemote(http))
12724
12804
  ]);
12725
12805
  const localBySlug = new Map(locals.map(l => [l.slug, l]));
12726
12806
  const remoteBySlug = new Map(remotes.map(r => [r.slug, r]));
@@ -12799,29 +12879,30 @@ async function buildPlan(modules, http, rootDir, options = {}) {
12799
12879
  }
12800
12880
  return { modules: modulePlans };
12801
12881
  }
12802
- async function applyPlan(plan, http, opts = {}) {
12882
+ async function applyPlan(plan, unlimitedHttp, opts = {}) {
12803
12883
  const concurrency = opts.concurrency ?? DEFAULT_CONCURRENCY;
12804
12884
  const reporter = opts.reporter ?? silentReporter;
12805
12885
  const errors = [];
12886
+ const abortController = new AbortController();
12887
+ const http = limitConcurrency(unlimitedHttp, concurrency, abortController.signal);
12888
+ let abortError;
12806
12889
  const handleErr = (label, e) => {
12807
- const msg = formatError(e);
12808
- const full = `${label}: ${msg}`;
12809
- if (opts.abortOnError)
12810
- throw new Error(full);
12890
+ if (abortError)
12891
+ throw abortError;
12892
+ const full = `${label}: ${formatError(e)}`;
12893
+ if (opts.abortOnError) {
12894
+ abortError = new Error(full);
12895
+ abortController.abort(abortError);
12896
+ throw abortError;
12897
+ }
12811
12898
  reporter.warn(full);
12812
12899
  errors.push(full);
12813
12900
  };
12814
12901
  for (const mp of plan.modules) {
12815
12902
  const { module: mod, creates, updates, deletes } = mp;
12816
- if (deletes.length) {
12817
- await runConcurrently(deletes.map(e => ({ slug: e.slug, run: () => mod.delete(http, e.remote.id) })), concurrency, `Deleting ${mod.displayName}`, handleErr, reporter);
12818
- }
12819
- if (updates.length) {
12820
- await runConcurrently(updates.map(e => ({ slug: e.slug, run: () => mod.update(http, e.local, e.remote.id) })), concurrency, `Updating ${mod.displayName}`, handleErr, reporter);
12821
- }
12822
- if (creates.length) {
12823
- await runConcurrently(creates.map(e => ({ slug: e.slug, run: () => mod.create(http, e.local) })), concurrency, `Creating ${mod.displayName}`, handleErr, reporter);
12824
- }
12903
+ await runAllAtOnce(deletes.map(e => ({ slug: e.slug, run: () => mod.delete(http, e.remote.id) })), `Deleting ${mod.displayName}`, handleErr, reporter);
12904
+ await runAllAtOnce(updates.map(e => ({ slug: e.slug, run: () => mod.update(http, e.local, e.remote.id) })), `Updating ${mod.displayName}`, handleErr, reporter);
12905
+ await runAllAtOnce(creates.map(e => ({ slug: e.slug, run: () => mod.create(http, e.local) })), `Creating ${mod.displayName}`, handleErr, reporter);
12825
12906
  }
12826
12907
  return { errors };
12827
12908
  }
@@ -12848,6 +12929,9 @@ function renderSchemaDetail(localData, remoteData, ignoredFields) {
12848
12929
  const remoteYaml = YAML.stringify(omit(remoteData, ignoredFields));
12849
12930
  return { schema: buildUnifiedDiff(remoteYaml, localYaml, "schema.yaml") };
12850
12931
  }
12932
+ function runAllAtOnce(tasks, label, onError, reporter) {
12933
+ return runConcurrently(tasks, tasks.length, label, onError, reporter);
12934
+ }
12851
12935
  async function runConcurrently(tasks, concurrency, label, onError, reporter) {
12852
12936
  if (!tasks.length)
12853
12937
  return;
@@ -12855,21 +12939,13 @@ async function runConcurrently(tasks, concurrency, label, onError, reporter) {
12855
12939
  let done = 0;
12856
12940
  const progress = reporter.progress(label, total);
12857
12941
  try {
12858
- const limit = Math.max(1, Math.min(total, concurrency));
12859
- let batch = [];
12860
- for (let i = 0; i < tasks.length; i++) {
12861
- batch.push(tasks[i]);
12862
- if (batch.length === limit || i === tasks.length - 1) {
12863
- await Promise.all(batch.map(task => task
12864
- .run()
12865
- .catch((e) => onError(`${label}: ${task.slug}`, e))
12866
- .finally(() => {
12867
- done++;
12868
- progress.update(task.slug, done, total);
12869
- })));
12870
- batch = [];
12871
- }
12872
- }
12942
+ await mapWithConcurrency(tasks, concurrency, task => task
12943
+ .run()
12944
+ .catch((e) => onError(`${label}: ${task.slug}`, e))
12945
+ .finally(() => {
12946
+ done++;
12947
+ progress.update(task.slug, done, total);
12948
+ }));
12873
12949
  progress.succeed();
12874
12950
  }
12875
12951
  catch (e) {
@@ -15726,14 +15802,6 @@ function normalizeSchema(schema) {
15726
15802
  }
15727
15803
  return normalized;
15728
15804
  }
15729
- async function mapInBatches(items, size, fn) {
15730
- const limit = Math.max(1, size);
15731
- const results = [];
15732
- for (let i = 0; i < items.length; i += limit) {
15733
- results.push(...(await Promise.all(items.slice(i, i + limit).map(fn))));
15734
- }
15735
- return results;
15736
- }
15737
15805
  function isNotFoundError(error) {
15738
15806
  if (!error || typeof error !== "object")
15739
15807
  return false;
@@ -15797,10 +15865,10 @@ const functionModule = {
15797
15865
  }
15798
15866
  return results;
15799
15867
  },
15800
- async readRemote(http, options = {}) {
15868
+ async readRemote(http) {
15801
15869
  const res = await http.get("function");
15802
15870
  const fns = unwrapList(res);
15803
- return mapInBatches(fns, options.concurrency ?? DEFAULT_CONCURRENCY, async (fn) => {
15871
+ return Promise.all(fns.map(async (fn) => {
15804
15872
  const id = fn._id;
15805
15873
  const [indexRes, depsRes] = await Promise.all([
15806
15874
  http.get(`function/${id}/index`).catch(emptyIfMissing({ index: "" })),
@@ -15823,7 +15891,7 @@ const functionModule = {
15823
15891
  dependencies
15824
15892
  }
15825
15893
  };
15826
- });
15894
+ }));
15827
15895
  },
15828
15896
  async create(http, local) {
15829
15897
  const id = await this.createSchema(http, local.data.schema);