@kubohiroya/sb3-toolchain 0.2.0 → 0.3.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -3,6 +3,15 @@
3
3
  import {cp, lstat, mkdtemp, readFile, rm, writeFile} from 'node:fs/promises';
4
4
  import path from 'node:path';
5
5
 
6
+ import {
7
+ compareExtensionApiManifests,
8
+ defaultExtensionApiManifestSizeLimit,
9
+ extensionApiManifestIntegrity,
10
+ formatExtensionApiCompatibilityChanges,
11
+ parseExtensionApiManifest,
12
+ validateExtensionApiManifestSourceMetadata,
13
+ validateManagedExtensionApiManifest,
14
+ } from './extension-api-manifest.js';
6
15
  import {
7
16
  extensionHeaderId,
8
17
  extensionIntegrity,
@@ -47,14 +56,34 @@ function githubHeaders(accept) {
47
56
  };
48
57
  }
49
58
 
50
- function rawArtifactUrl(source, commit) {
51
- const artifact = source.artifact
59
+ function rawArtifactUrl(source, commit, artifactPath = source.artifact) {
60
+ const artifact = artifactPath
52
61
  .split('/')
53
62
  .map((segment) => encodeURIComponent(segment))
54
63
  .join('/');
55
64
  return `https://raw.githubusercontent.com/${source.repository}/${commit}/${artifact}`;
56
65
  }
57
66
 
67
+ async function downloadExtensionApiManifest(
68
+ extension,
69
+ commit,
70
+ fetchImplementation,
71
+ maximumManifestBytes,
72
+ expectedId = extension.id,
73
+ ) {
74
+ const metadata = validateExtensionApiManifestSourceMetadata(extension);
75
+ if (!metadata) return null;
76
+ const contents = await fetchBytes(
77
+ fetchImplementation,
78
+ rawArtifactUrl(extension.source, commit, metadata.artifact),
79
+ maximumManifestBytes,
80
+ `GitHub extension API manifest download for ${extension.id}`,
81
+ 'application/json',
82
+ );
83
+ const manifest = parseExtensionApiManifest(contents, {expectedId});
84
+ return {contents, manifest};
85
+ }
86
+
58
87
  function githubCommitUrl(source) {
59
88
  return (
60
89
  `https://api.github.com/repos/${source.repository}/commits/` + encodeURIComponent(source.ref)
@@ -240,9 +269,12 @@ async function installCandidate({
240
269
  }
241
270
 
242
271
  async function updateCandidate({
272
+ allowBreakingApi,
273
+ apiManifestArtifact,
243
274
  confirmReplace,
244
275
  fetchImplementation,
245
276
  maximumArtifactBytes,
277
+ maximumManifestBytes,
246
278
  migrateToId,
247
279
  mode,
248
280
  selectedExtensionId,
@@ -254,6 +286,11 @@ async function updateCandidate({
254
286
  Number.isSafeInteger(maximumArtifactBytes) && maximumArtifactBytes > 0,
255
287
  'maximumArtifactBytes must be a positive integer.',
256
288
  );
289
+ assert(
290
+ Number.isSafeInteger(maximumManifestBytes) && maximumManifestBytes > 0,
291
+ 'maximumManifestBytes must be a positive integer.',
292
+ );
293
+ assert(!allowBreakingApi || yes, '--allow-breaking-api requires --yes.');
257
294
  const source = await inspectExtensionSource(sourceDirectory, {willReplace: true});
258
295
  const initialComparison = await compareDirectories(
259
296
  source.resolvedSourceDirectory,
@@ -284,6 +321,10 @@ async function updateCandidate({
284
321
  sourceArtifact === undefined || migrateToId !== undefined,
285
322
  'A replacement artifact path requires an extension ID migration.',
286
323
  );
324
+ assert(
325
+ apiManifestArtifact === undefined || migrateToId !== undefined,
326
+ 'A replacement API manifest artifact path requires an extension ID migration.',
327
+ );
287
328
  const selected = managed.filter(
288
329
  (extension) => selectedExtensionId === undefined || extension.id === selectedExtensionId,
289
330
  );
@@ -295,6 +336,7 @@ async function updateCandidate({
295
336
  ),
296
337
  );
297
338
  rewriteExtensionIdDocuments({
339
+ apiManifestArtifact,
298
340
  extensionManifest,
299
341
  newId: migrateToId,
300
342
  oldId: selectedExtensionId,
@@ -305,31 +347,96 @@ async function updateCandidate({
305
347
 
306
348
  const downloads = await Promise.all(
307
349
  selected.map(async (extension) => {
308
- const effectiveExtension =
309
- sourceArtifact === undefined
310
- ? extension
311
- : {
312
- ...extension,
313
- source: {...extension.source, artifact: sourceArtifact},
314
- };
350
+ let effectiveExtension = extension;
351
+ if (sourceArtifact !== undefined || apiManifestArtifact !== undefined) {
352
+ const effectiveSource = {...extension.source};
353
+ if (sourceArtifact !== undefined) effectiveSource.artifact = sourceArtifact;
354
+ if (apiManifestArtifact !== undefined) {
355
+ assert(
356
+ effectiveSource.apiManifest,
357
+ `Extension ${extension.id} has no managed API manifest metadata.`,
358
+ );
359
+ effectiveSource.apiManifest = {
360
+ ...effectiveSource.apiManifest,
361
+ artifact: apiManifestArtifact,
362
+ };
363
+ }
364
+ effectiveExtension = {...extension, source: effectiveSource};
365
+ }
315
366
  validateExtensionSourceMetadata(effectiveExtension);
316
367
  const commit =
317
368
  mode === 'sync'
318
369
  ? extension.source.resolvedCommit
319
370
  : await resolveGithubCommit(extension, fetchImplementation);
320
- const contents = await downloadExtension(
321
- effectiveExtension,
322
- commit,
323
- fetchImplementation,
324
- maximumArtifactBytes,
325
- migrateToId ?? extension.id,
326
- );
371
+ const expectedId = migrateToId ?? extension.id;
372
+ const [contents, apiManifestDownload] = await Promise.all([
373
+ downloadExtension(
374
+ effectiveExtension,
375
+ commit,
376
+ fetchImplementation,
377
+ maximumArtifactBytes,
378
+ expectedId,
379
+ ),
380
+ downloadExtensionApiManifest(
381
+ effectiveExtension,
382
+ commit,
383
+ fetchImplementation,
384
+ maximumManifestBytes,
385
+ expectedId,
386
+ ),
387
+ ]);
388
+ let compatibilityChanges = [];
327
389
  if (mode === 'sync') {
328
390
  validateManagedExtensionContents(extension, contents);
391
+ if (apiManifestDownload) {
392
+ validateManagedExtensionApiManifest(extension, apiManifestDownload.contents);
393
+ }
394
+ } else if (apiManifestDownload) {
395
+ const installedApiManifestContents = source.extensionApiManifestContents.get(extension.id);
396
+ assert(
397
+ installedApiManifestContents,
398
+ `Managed extension API manifest is missing for ${extension.id}.`,
399
+ );
400
+ const installedApiManifest = validateManagedExtensionApiManifest(
401
+ extension,
402
+ installedApiManifestContents,
403
+ ).manifest;
404
+ compatibilityChanges = compareExtensionApiManifests(
405
+ installedApiManifest,
406
+ apiManifestDownload.manifest,
407
+ );
329
408
  }
330
- return {commit, contents, effectiveExtension, extension};
409
+ return {
410
+ apiManifestDownload,
411
+ commit,
412
+ compatibilityChanges,
413
+ contents,
414
+ effectiveExtension,
415
+ extension,
416
+ };
331
417
  }),
332
418
  );
419
+ const breakingApiChanges = downloads.flatMap((download) =>
420
+ download.compatibilityChanges
421
+ .filter((change) => change.breaking)
422
+ .map((change) => ({...change, extensionId: download.extension.id})),
423
+ );
424
+ if (breakingApiChanges.length > 0 && !allowBreakingApi) {
425
+ const details = downloads
426
+ .filter((download) => download.compatibilityChanges.some((change) => change.breaking))
427
+ .map((download) =>
428
+ formatExtensionApiCompatibilityChanges(
429
+ download.extension.id,
430
+ download.compatibilityChanges.filter((change) => change.breaking),
431
+ ),
432
+ )
433
+ .join('\n');
434
+ throw new Error(
435
+ `Extension API update contains ${breakingApiChanges.length} breaking change(s). ` +
436
+ 'Review the reported paths, then use --allow-breaking-api with --yes to apply.\n' +
437
+ details,
438
+ );
439
+ }
333
440
 
334
441
  const parentDirectory = path.dirname(source.resolvedSourceDirectory);
335
442
  const candidateDirectory = await mkdtemp(
@@ -348,10 +455,17 @@ async function updateCandidate({
348
455
  const entriesById = new Map(manifest.extensions.map((extension) => [extension.id, extension]));
349
456
  let manifestChanged = false;
350
457
  let migration;
351
- for (const {commit, contents, effectiveExtension, extension} of downloads) {
458
+ for (const {
459
+ apiManifestDownload,
460
+ commit,
461
+ contents,
462
+ effectiveExtension,
463
+ extension,
464
+ } of downloads) {
352
465
  if (migrateToId !== undefined) {
353
466
  const project = JSON.parse(await readFile(projectPath, 'utf8'));
354
467
  migration = rewriteExtensionIdDocuments({
468
+ apiManifestArtifact,
355
469
  extensionManifest: manifest,
356
470
  newId: migrateToId,
357
471
  oldId: extension.id,
@@ -361,15 +475,38 @@ async function updateCandidate({
361
475
  const migratedExtension = migration.extensionManifest.extensions[migration.extensionIndex];
362
476
  migratedExtension.source.resolvedCommit = commit;
363
477
  migratedExtension.source.integrity = extensionIntegrity(contents);
364
- await Promise.all([
478
+ if (apiManifestDownload) {
479
+ migratedExtension.source.apiManifest.integrity = extensionApiManifestIntegrity(
480
+ apiManifestDownload.contents,
481
+ );
482
+ }
483
+ const writes = [
365
484
  writeFile(projectPath, `${JSON.stringify(migration.project, null, 2)}\n`),
366
485
  writeFile(manifestPath, `${JSON.stringify(migration.extensionManifest, null, 2)}\n`),
367
486
  writeFile(path.join(candidateDirectory, migration.newPath), contents),
368
- ]);
487
+ ];
488
+ if (apiManifestDownload && migration.newApiManifestPath) {
489
+ writes.push(
490
+ writeFile(
491
+ path.join(candidateDirectory, migration.newApiManifestPath),
492
+ apiManifestDownload.contents,
493
+ ),
494
+ );
495
+ }
496
+ await Promise.all(writes);
369
497
  await rm(path.join(candidateDirectory, migration.oldPath));
498
+ if (migration.oldApiManifestPath) {
499
+ await rm(path.join(candidateDirectory, migration.oldApiManifestPath));
500
+ }
370
501
  continue;
371
502
  }
372
503
  await writeFile(path.join(candidateDirectory, extension.path), contents);
504
+ if (apiManifestDownload) {
505
+ await writeFile(
506
+ path.join(candidateDirectory, extension.source.apiManifest.path),
507
+ apiManifestDownload.contents,
508
+ );
509
+ }
373
510
  if (mode === 'update') {
374
511
  const candidateExtension = entriesById.get(extension.id);
375
512
  assert(candidateExtension, `Extension manifest entry disappeared: ${extension.id}`);
@@ -380,6 +517,15 @@ async function updateCandidate({
380
517
  candidateExtension.source.integrity !== integrity;
381
518
  candidateExtension.source.resolvedCommit = commit;
382
519
  candidateExtension.source.integrity = integrity;
520
+ if (apiManifestDownload) {
521
+ const apiManifestIntegrity = extensionApiManifestIntegrity(apiManifestDownload.contents);
522
+ manifestChanged =
523
+ manifestChanged ||
524
+ candidateExtension.source.apiManifest.integrity !== apiManifestIntegrity;
525
+ candidateExtension.source.apiManifest.integrity = apiManifestIntegrity;
526
+ candidateExtension.source.apiManifest.formatVersion =
527
+ apiManifestDownload.manifest.formatVersion;
528
+ }
383
529
  }
384
530
  }
385
531
  if (manifestChanged && migrateToId === undefined) {
@@ -400,6 +546,13 @@ async function updateCandidate({
400
546
  installed = true;
401
547
  return {
402
548
  ...result,
549
+ apiCompatibility: downloads
550
+ .filter(({apiManifestDownload}) => apiManifestDownload !== null)
551
+ .map(({compatibilityChanges, extension}) => ({
552
+ changes: compatibilityChanges,
553
+ id: migrateToId ?? extension.id,
554
+ previousId: migrateToId === undefined ? undefined : extension.id,
555
+ })),
403
556
  extensions: downloads.map(({commit, extension}) => ({
404
557
  id: migrateToId ?? extension.id,
405
558
  previousId: migrateToId === undefined ? undefined : extension.id,
@@ -437,6 +590,12 @@ export async function extensionStatus(
437
590
  let local = 'valid';
438
591
  try {
439
592
  validateManagedExtensionContents(extension, source.extensionContents.get(extension.id));
593
+ if (extension.source.apiManifest) {
594
+ validateManagedExtensionApiManifest(
595
+ extension,
596
+ source.extensionApiManifestContents.get(extension.id),
597
+ );
598
+ }
440
599
  } catch {
441
600
  local = 'modified';
442
601
  }
@@ -456,13 +615,17 @@ export async function syncExtensions({
456
615
  confirmReplace,
457
616
  fetch: fetchImplementation = globalThis.fetch,
458
617
  maximumArtifactBytes = defaultExtensionArtifactSizeLimit,
618
+ maximumManifestBytes = defaultExtensionApiManifestSizeLimit,
459
619
  sourceDirectory,
460
620
  yes = false,
461
621
  }) {
462
622
  return updateCandidate({
623
+ allowBreakingApi: false,
624
+ apiManifestArtifact: undefined,
463
625
  confirmReplace,
464
626
  fetchImplementation: assertFetch(fetchImplementation),
465
627
  maximumArtifactBytes,
628
+ maximumManifestBytes,
466
629
  migrateToId: undefined,
467
630
  mode: 'sync',
468
631
  selectedExtensionId: undefined,
@@ -473,19 +636,25 @@ export async function syncExtensions({
473
636
  }
474
637
 
475
638
  export async function updateExtensions({
639
+ allowBreakingApi = false,
640
+ apiManifestArtifact,
476
641
  confirmReplace,
477
642
  extensionId,
478
643
  fetch: fetchImplementation = globalThis.fetch,
479
644
  maximumArtifactBytes = defaultExtensionArtifactSizeLimit,
645
+ maximumManifestBytes = defaultExtensionApiManifestSizeLimit,
480
646
  migrateToId,
481
647
  sourceArtifact,
482
648
  sourceDirectory,
483
649
  yes = false,
484
650
  }) {
485
651
  return updateCandidate({
652
+ allowBreakingApi,
653
+ apiManifestArtifact,
486
654
  confirmReplace,
487
655
  fetchImplementation: assertFetch(fetchImplementation),
488
656
  maximumArtifactBytes,
657
+ maximumManifestBytes,
489
658
  migrateToId,
490
659
  mode: 'update',
491
660
  selectedExtensionId: extensionId,
package/src/import.js CHANGED
@@ -7,6 +7,7 @@ import process from 'node:process';
7
7
  import {strFromU8, unzipSync} from 'fflate';
8
8
 
9
9
  import {validateArchiveEntryName} from './archive.js';
10
+ import {validateManagedExtensionApiManifest} from './extension-api-manifest.js';
10
11
  import {validateManagedExtensionContents} from './extension-dependencies.js';
11
12
  import {
12
13
  assertNoInterruptedRollback,
@@ -153,18 +154,20 @@ async function readExistingExtensionSources(outputDirectory) {
153
154
  `Existing embedded extension manifest is invalid: ${manifestPath}`,
154
155
  );
155
156
 
156
- return new Map(
157
- manifest.extensions
158
- .filter(
159
- (extension) =>
160
- extension &&
161
- typeof extension === 'object' &&
162
- typeof extension.id === 'string' &&
163
- typeof extension.path === 'string' &&
164
- extension.source !== undefined,
165
- )
166
- .map((extension) => [`${extension.id}\u0000${extension.path}`, extension.source]),
167
- );
157
+ const sources = new Map();
158
+ for (const extension of manifest.extensions) {
159
+ if (
160
+ !extension ||
161
+ typeof extension !== 'object' ||
162
+ typeof extension.id !== 'string' ||
163
+ typeof extension.path !== 'string' ||
164
+ extension.source === undefined
165
+ ) {
166
+ continue;
167
+ }
168
+ sources.set(`${extension.id}\u0000${extension.path}`, extension.source);
169
+ }
170
+ return sources;
168
171
  }
169
172
 
170
173
  export function decodeExtensionDataUrl(dataUrl) {
@@ -275,12 +278,25 @@ export async function importSb3({
275
278
  encoding: decoded.encoding,
276
279
  };
277
280
  const existingSource = existingExtensionSources.get(`${extensionId}\u0000${sourcePath}`);
281
+ let existingApiManifestContents = null;
278
282
  if (existingSource !== undefined) {
279
283
  extension.source = structuredClone(existingSource);
280
284
  validateManagedExtensionContents(extension, decoded.source);
285
+ if (extension.source.apiManifest) {
286
+ existingApiManifestContents = await readFile(
287
+ path.join(resolvedOutputDirectory, extension.source.apiManifest.path),
288
+ );
289
+ validateManagedExtensionApiManifest(extension, existingApiManifestContents);
290
+ }
281
291
  }
282
292
  embeddedExtensions.push(extension);
283
293
  decodedExtensionSources.push({path: sourcePath, source: decoded.source});
294
+ if (existingApiManifestContents) {
295
+ decodedExtensionSources.push({
296
+ path: extension.source.apiManifest.path,
297
+ source: existingApiManifestContents,
298
+ });
299
+ }
284
300
  }
285
301
 
286
302
  const outputParent = path.dirname(resolvedOutputDirectory);
package/src/index.js CHANGED
@@ -10,6 +10,17 @@ export {
10
10
  validateExtensionSourceMetadata,
11
11
  validateManagedExtensionContents,
12
12
  } from './extension-dependencies.js';
13
+ export {
14
+ compareExtensionApiManifests,
15
+ defaultExtensionApiManifestSizeLimit,
16
+ extensionApiManifestFormatVersion,
17
+ extensionApiManifestIntegrity,
18
+ extensionApiManifestLocalPath,
19
+ formatExtensionApiCompatibilityChanges,
20
+ parseExtensionApiManifest,
21
+ validateExtensionApiManifestSourceMetadata,
22
+ validateManagedExtensionApiManifest,
23
+ } from './extension-api-manifest.js';
13
24
  export {
14
25
  bundleExtensions,
15
26
  planExtensionBundle,
package/src/source.js CHANGED
@@ -7,6 +7,7 @@ import path from 'node:path';
7
7
  import {strToU8, zipSync} from 'fflate';
8
8
 
9
9
  import {validateArchiveEntryName} from './archive.js';
10
+ import {validateManagedExtensionApiManifest} from './extension-api-manifest.js';
10
11
  import {buildExtensionBundles, validateExtensionBundleConfigurations} from './extension-bundle.js';
11
12
  import {
12
13
  validateExtensionSourceMetadata,
@@ -194,6 +195,9 @@ function validateExtensionManifest(extensionManifest, project, extensionFiles) {
194
195
  validateExtensionSourceMetadata(extension);
195
196
  extensionsById.set(extension.id, extension);
196
197
  expectedFiles.push(`${extension.id}.js`);
198
+ if (extension.source?.apiManifest) {
199
+ expectedFiles.push(path.posix.relative('extensions', extension.source.apiManifest.path));
200
+ }
197
201
  }
198
202
 
199
203
  for (const [extensionId, extensionUrl] of Object.entries(extensionUrls)) {
@@ -334,6 +338,7 @@ async function inspectSb3Source(sourceDirectory, validateManagedExtensions) {
334
338
  );
335
339
 
336
340
  const extensionContents = new Map();
341
+ const extensionApiManifestContents = new Map();
337
342
  await Promise.all(
338
343
  extensions.map(async (extension) => {
339
344
  const contents = await readFile(path.join(resolvedSourceDirectory, extension.path));
@@ -341,6 +346,15 @@ async function inspectSb3Source(sourceDirectory, validateManagedExtensions) {
341
346
  validateManagedExtensionContents(extension, contents);
342
347
  }
343
348
  extensionContents.set(extension.id, contents);
349
+ if (extension.source?.apiManifest) {
350
+ const apiManifestContents = await readFile(
351
+ path.join(resolvedSourceDirectory, extension.source.apiManifest.path),
352
+ );
353
+ if (validateManagedExtensions) {
354
+ validateManagedExtensionApiManifest(extension, apiManifestContents);
355
+ }
356
+ extensionApiManifestContents.set(extension.id, apiManifestContents);
357
+ }
344
358
  }),
345
359
  );
346
360
 
@@ -348,6 +362,7 @@ async function inspectSb3Source(sourceDirectory, validateManagedExtensions) {
348
362
  assetContents,
349
363
  assetReferenceCount: references.length,
350
364
  extensions,
365
+ extensionApiManifestContents,
351
366
  extensionBundles,
352
367
  extensionContents,
353
368
  project,