@adea-ai/plugins 4.0.0 → 4.0.6
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/agent-plugins-catalog.d.ts +2 -4
- package/dist/agent-plugins-catalog.js +42 -50
- package/dist/agent-plugins-catalog.js.map +1 -1
- package/dist/agent-plugins-schema.d.ts +8 -0
- package/dist/agent-plugins-schema.js +10 -1
- package/dist/agent-plugins-schema.js.map +1 -1
- package/dist/artifacts.d.ts +57 -0
- package/dist/artifacts.js +159 -0
- package/dist/artifacts.js.map +1 -0
- package/dist/build.d.ts +20 -0
- package/dist/build.js +103 -0
- package/dist/build.js.map +1 -0
- package/dist/capabilities.d.ts +7 -0
- package/dist/capabilities.js +116 -0
- package/dist/capabilities.js.map +1 -0
- package/dist/catalog-store.d.ts +76 -0
- package/dist/catalog-store.js +463 -0
- package/dist/catalog-store.js.map +1 -0
- package/dist/concurrency.d.ts +1 -0
- package/dist/concurrency.js +15 -0
- package/dist/concurrency.js.map +1 -0
- package/dist/consumer-index.d.ts +7 -29
- package/dist/consumer-index.js +24 -15
- package/dist/consumer-index.js.map +1 -1
- package/dist/digest.d.ts +13 -0
- package/dist/digest.js +47 -0
- package/dist/digest.js.map +1 -0
- package/dist/fs-atomic.d.ts +7 -0
- package/dist/fs-atomic.js +30 -0
- package/dist/fs-atomic.js.map +1 -0
- package/dist/harness.js +2 -1
- package/dist/harness.js.map +1 -1
- package/dist/icon-mirror.d.ts +5 -4
- package/dist/icon-mirror.js +73 -72
- package/dist/icon-mirror.js.map +1 -1
- package/dist/icons.d.ts +2 -2
- package/dist/icons.js +4 -12
- package/dist/icons.js.map +1 -1
- package/dist/index.d.ts +16 -309
- package/dist/index.js +12 -1900
- package/dist/index.js.map +1 -1
- package/dist/integrity-manifest.d.ts +43 -13
- package/dist/integrity-manifest.js +28 -17
- package/dist/integrity-manifest.js.map +1 -1
- package/dist/json-values.d.ts +3 -0
- package/dist/json-values.js +10 -0
- package/dist/json-values.js.map +1 -0
- package/dist/plugin-release.d.ts +38 -0
- package/dist/plugin-release.js +321 -0
- package/dist/plugin-release.js.map +1 -0
- package/dist/plugin-skips.d.ts +19 -0
- package/dist/plugin-skips.js +65 -0
- package/dist/plugin-skips.js.map +1 -0
- package/dist/policy.d.ts +13 -0
- package/dist/policy.js +26 -0
- package/dist/policy.js.map +1 -0
- package/dist/portable-catalog.d.ts +4 -4
- package/dist/portable-catalog.js +14 -126
- package/dist/portable-catalog.js.map +1 -1
- package/dist/portable-verify.d.ts +2 -0
- package/dist/portable-verify.js +68 -0
- package/dist/portable-verify.js.map +1 -0
- package/dist/publication.d.ts +12 -12
- package/dist/publication.js +31 -21
- package/dist/publication.js.map +1 -1
- package/dist/published-output.d.ts +17 -0
- package/dist/published-output.js +49 -0
- package/dist/published-output.js.map +1 -0
- package/dist/site-icons.d.ts +4 -3
- package/dist/site-icons.js +38 -11
- package/dist/site-icons.js.map +1 -1
- package/dist/snapshots.d.ts +39 -0
- package/dist/snapshots.js +238 -0
- package/dist/snapshots.js.map +1 -0
- package/dist/source-resolution.d.ts +28 -0
- package/dist/source-resolution.js +158 -0
- package/dist/source-resolution.js.map +1 -0
- package/dist/sources.d.ts +5 -1
- package/dist/sources.js +21 -27
- package/dist/sources.js.map +1 -1
- package/dist/synchronize.d.ts +44 -0
- package/dist/synchronize.js +118 -0
- package/dist/synchronize.js.map +1 -0
- package/dist/taxonomy.d.ts +23 -0
- package/dist/taxonomy.js +62 -0
- package/dist/taxonomy.js.map +1 -0
- package/dist/upstream.d.ts +42 -0
- package/dist/upstream.js +179 -0
- package/dist/upstream.js.map +1 -0
- package/dist/verify-artifacts.d.ts +36 -0
- package/dist/verify-artifacts.js +286 -0
- package/dist/verify-artifacts.js.map +1 -0
- package/native/build-provenance.txt +5 -5
- package/native/staged-reader-win32-arm64.node +0 -0
- package/native/staged-reader-win32-x64.node +0 -0
- package/package.json +3 -3
- package/src/agent-plugins-catalog.test.ts +2 -2
- package/src/agent-plugins-catalog.ts +46 -49
- package/src/agent-plugins-schema.ts +11 -1
- package/src/artifacts.test.ts +80 -0
- package/src/artifacts.ts +234 -0
- package/src/build.ts +131 -0
- package/src/capabilities.ts +153 -0
- package/src/catalog-store.ts +519 -0
- package/src/concurrency.ts +19 -0
- package/src/consumer-index.test.ts +147 -16
- package/src/consumer-index.ts +35 -43
- package/src/default-head.test.ts +143 -0
- package/src/digest.ts +55 -0
- package/src/fs-atomic.ts +32 -0
- package/src/harness.test.ts +4 -32
- package/src/harness.ts +2 -1
- package/src/icon-mirror.ts +78 -72
- package/src/icons.test.ts +0 -4
- package/src/icons.ts +4 -4
- package/src/index.test.ts +172 -46
- package/src/index.ts +58 -2562
- package/src/integrity-manifest.ts +41 -21
- package/src/json-values.ts +11 -0
- package/src/manifest-retry.test.ts +49 -0
- package/src/plugin-release.ts +457 -0
- package/src/plugin-skips.ts +110 -0
- package/src/policy.ts +39 -0
- package/src/portable-catalog.test.ts +0 -5
- package/src/portable-catalog.ts +16 -143
- package/src/portable-verify.ts +71 -0
- package/src/publication.test.ts +29 -1
- package/src/publication.ts +38 -22
- package/src/published-output.ts +60 -0
- package/src/site-icons.test.ts +73 -1
- package/src/site-icons.ts +31 -14
- package/src/snapshots.ts +287 -0
- package/src/source-config-sync.test.ts +84 -0
- package/src/source-digest.test.ts +61 -8
- package/src/source-resolution.ts +233 -0
- package/src/sources-json.test.ts +18 -2
- package/src/sources.ts +27 -33
- package/src/synchronize.ts +213 -0
- package/src/taxonomy.ts +87 -0
- package/src/upstream.ts +194 -0
- package/src/verify-artifacts.ts +365 -0
- package/src/write-artifacts.test.ts +374 -0
- package/dist/source-digest.d.ts +0 -2
- package/dist/source-digest.js +0 -12
- package/dist/source-digest.js.map +0 -1
- package/native/README.md +0 -24
- package/src/source-digest.ts +0 -14
package/src/site-icons.test.ts
CHANGED
|
@@ -32,7 +32,6 @@ function fetcher(site: Record<string, string | Uint8Array>): SiteIconFetcher & {
|
|
|
32
32
|
return {
|
|
33
33
|
textCalls,
|
|
34
34
|
byteCalls,
|
|
35
|
-
digest: (bytes) => byteDigest(bytes),
|
|
36
35
|
fetchText: async (url) => {
|
|
37
36
|
textCalls.push(url)
|
|
38
37
|
const value = site[url]
|
|
@@ -96,6 +95,68 @@ describe('site icon discovery', () => {
|
|
|
96
95
|
])
|
|
97
96
|
})
|
|
98
97
|
|
|
98
|
+
test('resolves relative icon links against the first document base URL', () => {
|
|
99
|
+
const baseHtml = `<!-- <base href="https://ignored.example/"> <script> -->
|
|
100
|
+
<script>const sample = '<!-- <base href="https://script.example/">'</script>
|
|
101
|
+
<meta content="<base href='https://attribute.example/'>">
|
|
102
|
+
<base target="_blank"><base href="/assets/icons/">
|
|
103
|
+
<base href="https://later.example/">
|
|
104
|
+
<link rel="icon" href="mark.png">`
|
|
105
|
+
expect(discoverSiteIcons(baseHtml, 'https://vendor.example/docs/page')).toEqual([
|
|
106
|
+
'https://vendor.example/assets/icons/mark.png',
|
|
107
|
+
'https://vendor.example/favicon.ico',
|
|
108
|
+
])
|
|
109
|
+
})
|
|
110
|
+
|
|
111
|
+
test('ignores bases and links inside text and template content', () => {
|
|
112
|
+
for (const inert of [
|
|
113
|
+
'<title><base href="https://title.example/"></title>',
|
|
114
|
+
'<textarea><base href="https://textarea.example/"></textarea>',
|
|
115
|
+
`<template><template><base href="https://nested.example/"></template>
|
|
116
|
+
<link rel="icon" href="https://template.example/icon.png"></template>`,
|
|
117
|
+
]) {
|
|
118
|
+
expect(
|
|
119
|
+
discoverSiteIcons(
|
|
120
|
+
`${inert}<base href="/real/"><link rel="icon" href="mark.png">`,
|
|
121
|
+
'https://vendor.example/'
|
|
122
|
+
)
|
|
123
|
+
).toEqual(['https://vendor.example/real/mark.png', 'https://vendor.example/favicon.ico'])
|
|
124
|
+
}
|
|
125
|
+
})
|
|
126
|
+
|
|
127
|
+
test('decodes HTML character references in URLs', () => {
|
|
128
|
+
expect(
|
|
129
|
+
discoverSiteIcons(
|
|
130
|
+
'<base href="/icons/?a=1&b=2"><link rel="icon" href="mark.png?x=1&y=2">',
|
|
131
|
+
'https://vendor.example/'
|
|
132
|
+
)
|
|
133
|
+
).toEqual([
|
|
134
|
+
'https://vendor.example/icons/mark.png?x=1&y=2',
|
|
135
|
+
'https://vendor.example/favicon.ico',
|
|
136
|
+
])
|
|
137
|
+
})
|
|
138
|
+
|
|
139
|
+
test('retains the homepage fallback and URL policy with a document base', () => {
|
|
140
|
+
expect(
|
|
141
|
+
discoverSiteIcons(
|
|
142
|
+
'<base href="https://cdn.example/icons/"><link rel="icon" href="mark.png">',
|
|
143
|
+
'https://vendor.example/docs/page'
|
|
144
|
+
)
|
|
145
|
+
).toEqual(['https://cdn.example/icons/mark.png', 'https://vendor.example/favicon.ico'])
|
|
146
|
+
expect(
|
|
147
|
+
discoverSiteIcons(
|
|
148
|
+
'<base href="http://vendor.example/assets/"><link rel="icon" href="mark.png">',
|
|
149
|
+
'https://vendor.example/docs/page'
|
|
150
|
+
)
|
|
151
|
+
).toEqual(['https://vendor.example/favicon.ico'])
|
|
152
|
+
expect(
|
|
153
|
+
discoverSiteIcons(
|
|
154
|
+
'<base href="http://["><base href="/later/"><link rel="icon" href="mark.png">',
|
|
155
|
+
'https://vendor.example/docs/page'
|
|
156
|
+
)
|
|
157
|
+
).toEqual(['https://vendor.example/docs/mark.png', 'https://vendor.example/favicon.ico'])
|
|
158
|
+
})
|
|
159
|
+
|
|
99
160
|
test('prefers the larger declared size within a group', () => {
|
|
100
161
|
const sizes = `<link rel="icon" sizes="32x32" href="/small.png">
|
|
101
162
|
<link rel="icon" sizes="512x512" href="/large.png">`
|
|
@@ -163,6 +224,17 @@ describe('site icon resolution', () => {
|
|
|
163
224
|
expect(io.byteCalls).toEqual(['https://vendor.example/apple-icon.png'])
|
|
164
225
|
})
|
|
165
226
|
|
|
227
|
+
test('fetches a relative icon from the document base', async () => {
|
|
228
|
+
const page = 'https://vendor.example/docs/page'
|
|
229
|
+
const baseHtml = '<head><base href="/assets/"><link rel="icon" href="mark.png"></head>'
|
|
230
|
+
const io = fetcher({ [page]: baseHtml, 'https://vendor.example/assets/mark.png': PNG })
|
|
231
|
+
expect(await resolveSiteIcon(page, io)).toMatchObject({
|
|
232
|
+
url: 'https://vendor.example/assets/mark.png',
|
|
233
|
+
homepage: page,
|
|
234
|
+
})
|
|
235
|
+
expect(io.byteCalls).toEqual(['https://vendor.example/assets/mark.png'])
|
|
236
|
+
})
|
|
237
|
+
|
|
166
238
|
test('accepts an icon container and skips a candidate that fails inspection', async () => {
|
|
167
239
|
const ico = fetcher({ [homepage]: html, 'https://vendor.example/apple-icon.png': ICO })
|
|
168
240
|
expect(await resolveSiteIcon(homepage, ico)).toMatchObject({ contentType: 'image/x-icon' })
|
package/src/site-icons.ts
CHANGED
|
@@ -1,3 +1,5 @@
|
|
|
1
|
+
import { parseFragment, type DefaultTreeAdapterTypes } from 'parse5'
|
|
2
|
+
import { byteDigest } from './digest.js'
|
|
1
3
|
import { inspectIconBytes, MAX_ICON_BYTES, type IconContentType } from './icons.js'
|
|
2
4
|
|
|
3
5
|
/**
|
|
@@ -5,8 +7,8 @@ import { inspectIconBytes, MAX_ICON_BYTES, type IconContentType } from './icons.
|
|
|
5
7
|
*
|
|
6
8
|
* Many plugins are a thin wrapper over a vendor's API: the content carries no
|
|
7
9
|
* logo, and the declared homepage is the only pointer to the vendor. Fetching
|
|
8
|
-
* that site's icon once at publish time, verifying it and mirroring it
|
|
9
|
-
*
|
|
10
|
+
* that site's icon once at publish time, verifying it and mirroring it beside
|
|
11
|
+
* the catalog replaces a per-render favicon lookup in the client.
|
|
10
12
|
*/
|
|
11
13
|
export const SITE_ICON_MAX_HTML_BYTES = 4 * 1024 * 1024
|
|
12
14
|
|
|
@@ -76,19 +78,35 @@ export function isPublicHostname(hostname: string): boolean {
|
|
|
76
78
|
* `apple-touch-icon` is preferred because it is a square PNG sized for a home
|
|
77
79
|
* screen; declared `icon` links follow in declared order; `/favicon.ico` is the
|
|
78
80
|
* conventional last resort when nothing is declared.
|
|
81
|
+
* Relative links use the document's first base href when present; the fallback
|
|
82
|
+
* stays on the homepage origin.
|
|
79
83
|
*/
|
|
80
84
|
export function discoverSiteIcons(html: string, baseUrl: string): readonly string[] {
|
|
81
85
|
const candidates: { href: string; rank: number }[] = []
|
|
82
|
-
const
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
))
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
|
|
86
|
+
const elements: DefaultTreeAdapterTypes.Element[] = []
|
|
87
|
+
const nodes: DefaultTreeAdapterTypes.ChildNode[] = parseFragment(html).childNodes.toReversed()
|
|
88
|
+
while (nodes.length > 0) {
|
|
89
|
+
const node = nodes.pop()!
|
|
90
|
+
if (!('tagName' in node)) continue
|
|
91
|
+
if (node.namespaceURI === 'http://www.w3.org/1999/xhtml') elements.push(node)
|
|
92
|
+
// Template contents are a separate, inert document fragment.
|
|
93
|
+
if (node.tagName !== 'template') nodes.push(...node.childNodes.toReversed())
|
|
94
|
+
}
|
|
95
|
+
let resolutionBase = baseUrl
|
|
96
|
+
for (const element of elements) {
|
|
97
|
+
if (element.tagName !== 'base') continue
|
|
98
|
+
const attributes = new Map(element.attrs.map(({ name, value }) => [name, value]))
|
|
99
|
+
if (!attributes.has('href')) continue
|
|
100
|
+
try {
|
|
101
|
+
resolutionBase = new URL(attributes.get('href')!, baseUrl).toString()
|
|
102
|
+
} catch {
|
|
103
|
+
// An invalid first base falls back to the document URL.
|
|
91
104
|
}
|
|
105
|
+
break
|
|
106
|
+
}
|
|
107
|
+
for (const element of elements) {
|
|
108
|
+
if (element.tagName !== 'link') continue
|
|
109
|
+
const attributes = new Map(element.attrs.map(({ name, value }) => [name, value]))
|
|
92
110
|
const rel = new Set((attributes.get('rel') ?? '').toLowerCase().split(/\s+/))
|
|
93
111
|
if (
|
|
94
112
|
!['apple-touch-icon', 'apple-touch-icon-precomposed', 'icon', 'mask-icon'].some((name) =>
|
|
@@ -112,7 +130,7 @@ export function discoverSiteIcons(html: string, baseUrl: string): readonly strin
|
|
|
112
130
|
const resolved: { url: string; rank: number }[] = []
|
|
113
131
|
for (const candidate of candidates.toSorted((left, right) => left.rank - right.rank)) {
|
|
114
132
|
try {
|
|
115
|
-
const url = new URL(candidate.href,
|
|
133
|
+
const url = new URL(candidate.href, resolutionBase)
|
|
116
134
|
if (url.protocol !== 'https:' || !isPublicHostname(url.hostname)) continue
|
|
117
135
|
resolved.push({ url: url.toString(), rank: candidate.rank })
|
|
118
136
|
} catch {
|
|
@@ -134,7 +152,6 @@ export function discoverSiteIcons(html: string, baseUrl: string): readonly strin
|
|
|
134
152
|
export interface SiteIconFetcher {
|
|
135
153
|
readonly fetchText: (url: string) => Promise<string>
|
|
136
154
|
readonly fetchBytes: (url: string) => Promise<Uint8Array>
|
|
137
|
-
readonly digest: (bytes: Uint8Array) => string
|
|
138
155
|
}
|
|
139
156
|
|
|
140
157
|
/**
|
|
@@ -171,7 +188,7 @@ export async function resolveSiteIcon(
|
|
|
171
188
|
homepage,
|
|
172
189
|
bytes,
|
|
173
190
|
contentType: inspected.contentType,
|
|
174
|
-
digest:
|
|
191
|
+
digest: byteDigest(bytes),
|
|
175
192
|
}
|
|
176
193
|
}
|
|
177
194
|
return { failed: 'NO_USABLE_ICON' }
|
package/src/snapshots.ts
ADDED
|
@@ -0,0 +1,287 @@
|
|
|
1
|
+
import { promises as fs } from 'node:fs'
|
|
2
|
+
import { join, relative, resolve } from 'node:path'
|
|
3
|
+
import { Readable } from 'node:stream'
|
|
4
|
+
import { createGunzip } from 'node:zlib'
|
|
5
|
+
import { extract } from 'tar-stream'
|
|
6
|
+
import type { CatalogPolicy } from './policy.js'
|
|
7
|
+
import { PluginSafetyError } from './plugin-skips.js'
|
|
8
|
+
import { canonicalRepositoryUrl, safeRelativePath, type SourceConfig } from './sources.js'
|
|
9
|
+
import { fetchUpstream, GITHUB_REPOSITORY_HOSTS, upstreamRequestHeaders } from './upstream.js'
|
|
10
|
+
|
|
11
|
+
export interface Snapshot {
|
|
12
|
+
readonly files: ReadonlyMap<string, Uint8Array>
|
|
13
|
+
readonly symlinks: readonly string[]
|
|
14
|
+
}
|
|
15
|
+
|
|
16
|
+
export interface SnapshotLoader {
|
|
17
|
+
load(repositoryUrl: string, commitSha: string, pluginSubdirectory: string): Promise<Snapshot>
|
|
18
|
+
}
|
|
19
|
+
|
|
20
|
+
/**
|
|
21
|
+
* Ceiling on bytes read out of one repository archive.
|
|
22
|
+
*
|
|
23
|
+
* The per-plugin policy still bounds what any single plugin may contain; this
|
|
24
|
+
* only stops a decompression bomb from exhausting the build host, and it is
|
|
25
|
+
* enforced while the stream is read rather than after it is buffered.
|
|
26
|
+
*/
|
|
27
|
+
const MAX_ARCHIVE_BYTES = 512 * 1024 * 1024
|
|
28
|
+
|
|
29
|
+
function concatChunks(chunks: readonly Uint8Array[], length: number): Uint8Array {
|
|
30
|
+
const out = new Uint8Array(length)
|
|
31
|
+
let offset = 0
|
|
32
|
+
for (const chunk of chunks) {
|
|
33
|
+
out.set(chunk, offset)
|
|
34
|
+
offset += chunk.byteLength
|
|
35
|
+
}
|
|
36
|
+
return out
|
|
37
|
+
}
|
|
38
|
+
|
|
39
|
+
/**
|
|
40
|
+
* Read one GitHub repository tarball into path -> bytes.
|
|
41
|
+
*
|
|
42
|
+
* GitHub wraps every entry in a single top-level `<repo>-<sha>/` directory,
|
|
43
|
+
* which is stripped so the keys match the tree API's repository-relative
|
|
44
|
+
* paths. Only regular files are taken: directories, symlinks, hardlinks and
|
|
45
|
+
* the `pax_global_header` record are skipped, so a link can never be
|
|
46
|
+
* materialized as content. Every remaining path goes through
|
|
47
|
+
* `safeRelativePath`, so an archive that tries to escape the repository is
|
|
48
|
+
* rejected rather than partially trusted. This is a whole-repository archive
|
|
49
|
+
* shared across plugins, so an oversized unrelated file is drained and
|
|
50
|
+
* counted against the archive budget but not retained. The selected plugin's
|
|
51
|
+
* files are checked against the tree before the archive is fetched and again
|
|
52
|
+
* when its snapshot is validated.
|
|
53
|
+
*/
|
|
54
|
+
export async function readArchiveFiles(
|
|
55
|
+
archive: Uint8Array,
|
|
56
|
+
policy: CatalogPolicy
|
|
57
|
+
): Promise<Map<string, Uint8Array>> {
|
|
58
|
+
const files = new Map<string, Uint8Array>()
|
|
59
|
+
let totalBytes = 0
|
|
60
|
+
const parser = extract()
|
|
61
|
+
const gunzip = createGunzip()
|
|
62
|
+
const settled = new Promise<void>((resolveDone, rejectDone) => {
|
|
63
|
+
const fail = (error: unknown): void => {
|
|
64
|
+
gunzip.destroy()
|
|
65
|
+
parser.destroy()
|
|
66
|
+
rejectDone(error)
|
|
67
|
+
}
|
|
68
|
+
parser.on('entry', (header, stream, next) => {
|
|
69
|
+
const skip = (): void => {
|
|
70
|
+
stream.on('end', next)
|
|
71
|
+
stream.resume()
|
|
72
|
+
}
|
|
73
|
+
if (header.type !== 'file') return skip()
|
|
74
|
+
const segments = header.name.split('/')
|
|
75
|
+
if (segments.length < 2) return skip()
|
|
76
|
+
let path: string
|
|
77
|
+
try {
|
|
78
|
+
path = safeRelativePath(segments.slice(1).join('/'), 'TREE_PATH_INVALID')
|
|
79
|
+
} catch (error) {
|
|
80
|
+
return fail(error)
|
|
81
|
+
}
|
|
82
|
+
const chunks: Uint8Array[] = []
|
|
83
|
+
let length = 0
|
|
84
|
+
stream.on('data', (chunk: unknown) => {
|
|
85
|
+
const bytes = chunk as Uint8Array
|
|
86
|
+
length += bytes.byteLength
|
|
87
|
+
if (totalBytes + length > MAX_ARCHIVE_BYTES) {
|
|
88
|
+
fail(new Error('PLUGIN_SIZE_POLICY: archive'))
|
|
89
|
+
return
|
|
90
|
+
}
|
|
91
|
+
if (length <= policy.maxFileBytes) chunks.push(bytes)
|
|
92
|
+
else chunks.length = 0
|
|
93
|
+
})
|
|
94
|
+
stream.on('error', fail)
|
|
95
|
+
stream.on('end', () => {
|
|
96
|
+
try {
|
|
97
|
+
totalBytes += length
|
|
98
|
+
if (length <= policy.maxFileBytes) files.set(path, concatChunks(chunks, length))
|
|
99
|
+
next()
|
|
100
|
+
} catch (error) {
|
|
101
|
+
fail(error)
|
|
102
|
+
}
|
|
103
|
+
})
|
|
104
|
+
stream.resume()
|
|
105
|
+
})
|
|
106
|
+
parser.on('error', fail)
|
|
107
|
+
gunzip.on('error', fail)
|
|
108
|
+
parser.on('finish', () => resolveDone())
|
|
109
|
+
})
|
|
110
|
+
Readable.from([archive]).pipe(gunzip).pipe(parser)
|
|
111
|
+
await settled
|
|
112
|
+
return files
|
|
113
|
+
}
|
|
114
|
+
|
|
115
|
+
export class NetworkSnapshotLoader implements SnapshotLoader {
|
|
116
|
+
readonly #policy: CatalogPolicy
|
|
117
|
+
readonly #trees = new Map<string, Promise<readonly TreeEntry[]>>()
|
|
118
|
+
readonly #archives = new Map<string, Promise<ReadonlyMap<string, Uint8Array>>>()
|
|
119
|
+
|
|
120
|
+
constructor(policy: CatalogPolicy) {
|
|
121
|
+
this.#policy = policy
|
|
122
|
+
}
|
|
123
|
+
|
|
124
|
+
async load(
|
|
125
|
+
repositoryUrl: string,
|
|
126
|
+
commitSha: string,
|
|
127
|
+
pluginSubdirectory: string
|
|
128
|
+
): Promise<Snapshot> {
|
|
129
|
+
const parsed = new URL(canonicalRepositoryUrl(repositoryUrl))
|
|
130
|
+
if (!GITHUB_REPOSITORY_HOSTS.includes(parsed.hostname))
|
|
131
|
+
throw new Error(`GIT_HOST_UNSUPPORTED: ${parsed.hostname}`)
|
|
132
|
+
const [owner, repo] = parsed.pathname.split('/').filter(Boolean)
|
|
133
|
+
if (!owner || !repo) throw new Error('GIT_REPOSITORY_INVALID')
|
|
134
|
+
const key = `${owner}/${repo}@${commitSha}`
|
|
135
|
+
let pendingTree = this.#trees.get(key)
|
|
136
|
+
if (!pendingTree) {
|
|
137
|
+
pendingTree = (async () => {
|
|
138
|
+
const treeUrl = `https://api.github.com/repos/${owner}/${repo}/git/trees/${commitSha}?recursive=1`
|
|
139
|
+
const response = await fetchUpstream(treeUrl, {
|
|
140
|
+
headers: upstreamRequestHeaders(treeUrl),
|
|
141
|
+
})
|
|
142
|
+
if (!response.ok)
|
|
143
|
+
throw new PluginSafetyError('PLUGIN_SOURCE_UNAVAILABLE', [`${owner}/${repo}`])
|
|
144
|
+
const payload = (await response.json()) as { truncated?: boolean; tree?: TreeEntry[] }
|
|
145
|
+
if (payload.truncated) throw new Error(`GIT_TREE_TRUNCATED: ${owner}/${repo}@${commitSha}`)
|
|
146
|
+
return (payload.tree ?? []).map((entry) => ({
|
|
147
|
+
...entry,
|
|
148
|
+
path: safeRelativePath(entry.path, 'TREE_PATH_INVALID'),
|
|
149
|
+
}))
|
|
150
|
+
})().catch((error: unknown) => {
|
|
151
|
+
this.#trees.delete(key)
|
|
152
|
+
throw error
|
|
153
|
+
})
|
|
154
|
+
this.#trees.set(key, pendingTree)
|
|
155
|
+
}
|
|
156
|
+
const tree = await pendingTree
|
|
157
|
+
const prefix = pluginSubdirectory ? `${safeRelativePath(pluginSubdirectory)}/` : ''
|
|
158
|
+
const submodules = tree.filter(
|
|
159
|
+
(entry) =>
|
|
160
|
+
entry.path.startsWith(prefix) && (entry.type === 'commit' || entry.mode === '160000')
|
|
161
|
+
)
|
|
162
|
+
if (submodules.length > 0)
|
|
163
|
+
throw new Error(`GIT_SUBMODULE_UNSUPPORTED: ${repositoryUrl}:${pluginSubdirectory}`)
|
|
164
|
+
const selected = tree.filter((entry) => entry.type === 'blob' && entry.path.startsWith(prefix))
|
|
165
|
+
if (selected.length === 0)
|
|
166
|
+
throw new Error(`PLUGIN_PATH_NOT_FOUND: ${repositoryUrl}:${pluginSubdirectory}`)
|
|
167
|
+
const files = new Map<string, Uint8Array>()
|
|
168
|
+
const symlinks = tree
|
|
169
|
+
.filter((entry) => entry.mode === '120000' && entry.path.startsWith(prefix))
|
|
170
|
+
.map((entry) => entry.path)
|
|
171
|
+
if (selected.length > this.#policy.maxFilesPerPlugin) throw new Error('PLUGIN_TOO_MANY_FILES')
|
|
172
|
+
let totalSize = 0
|
|
173
|
+
for (const entry of selected) {
|
|
174
|
+
const size = entry.size ?? 0
|
|
175
|
+
if (size > this.#policy.maxFileBytes || totalSize + size > this.#policy.maxBytesPerPlugin)
|
|
176
|
+
throw new Error(`PLUGIN_SIZE_POLICY: ${entry.path}`)
|
|
177
|
+
totalSize += size
|
|
178
|
+
}
|
|
179
|
+
// The tree above already bounded this plugin before a byte was pulled, so
|
|
180
|
+
// the archive is the only unbounded read left and it is capped while it
|
|
181
|
+
// streams. One archive per repository replaces one request per file.
|
|
182
|
+
const archive = await this.#archiveFiles(key, owner, repo, commitSha)
|
|
183
|
+
for (const entry of selected) {
|
|
184
|
+
const bytes = archive.get(entry.path)
|
|
185
|
+
if (!bytes) throw new PluginSafetyError('PLUGIN_FILE_UNAVAILABLE', [entry.path])
|
|
186
|
+
if (bytes.byteLength > this.#policy.maxFileBytes)
|
|
187
|
+
throw new Error(`PLUGIN_FILE_TOO_LARGE: ${entry.path}`)
|
|
188
|
+
files.set(
|
|
189
|
+
prefix ? relative(prefix.slice(0, -1), entry.path).replaceAll('\\', '/') : entry.path,
|
|
190
|
+
bytes
|
|
191
|
+
)
|
|
192
|
+
}
|
|
193
|
+
return { files, symlinks }
|
|
194
|
+
}
|
|
195
|
+
|
|
196
|
+
async #archiveFiles(
|
|
197
|
+
key: string,
|
|
198
|
+
owner: string,
|
|
199
|
+
repo: string,
|
|
200
|
+
commitSha: string
|
|
201
|
+
): Promise<ReadonlyMap<string, Uint8Array>> {
|
|
202
|
+
const cached = this.#archives.get(key)
|
|
203
|
+
if (cached) return cached
|
|
204
|
+
const pending = (async () => {
|
|
205
|
+
const url = `https://api.github.com/repos/${owner}/${repo}/tarball/${commitSha}`
|
|
206
|
+
const response = await fetchUpstream(url, { headers: upstreamRequestHeaders(url) })
|
|
207
|
+
if (!response.ok)
|
|
208
|
+
throw new PluginSafetyError('PLUGIN_SOURCE_UNAVAILABLE', [`${owner}/${repo}`])
|
|
209
|
+
return readArchiveFiles(new Uint8Array(await response.arrayBuffer()), this.#policy)
|
|
210
|
+
})().catch((error: unknown) => {
|
|
211
|
+
this.#archives.delete(key)
|
|
212
|
+
throw error
|
|
213
|
+
})
|
|
214
|
+
this.#archives.set(key, pending)
|
|
215
|
+
return pending
|
|
216
|
+
}
|
|
217
|
+
}
|
|
218
|
+
|
|
219
|
+
interface TreeEntry {
|
|
220
|
+
readonly path: string
|
|
221
|
+
readonly mode?: string
|
|
222
|
+
readonly type?: string
|
|
223
|
+
readonly size?: number
|
|
224
|
+
}
|
|
225
|
+
|
|
226
|
+
export class FixtureSnapshotLoader implements SnapshotLoader {
|
|
227
|
+
readonly #root: string
|
|
228
|
+
readonly #sources: ReadonlyMap<string, SourceConfig>
|
|
229
|
+
|
|
230
|
+
constructor(root: string, sources: readonly SourceConfig[]) {
|
|
231
|
+
this.#root = resolve(root)
|
|
232
|
+
this.#sources = new Map(
|
|
233
|
+
sources.map((source) => [canonicalRepositoryUrl(source.repositoryUrl), source])
|
|
234
|
+
)
|
|
235
|
+
}
|
|
236
|
+
|
|
237
|
+
async load(
|
|
238
|
+
repositoryUrl: string,
|
|
239
|
+
_commitSha: string,
|
|
240
|
+
pluginSubdirectory: string
|
|
241
|
+
): Promise<Snapshot> {
|
|
242
|
+
const source = this.#sources.get(canonicalRepositoryUrl(repositoryUrl))
|
|
243
|
+
if (!source?.fixturePath) throw new Error(`FIXTURE_REPOSITORY_NOT_FOUND: ${repositoryUrl}`)
|
|
244
|
+
return snapshotFromDirectory(join(this.#root, source.fixturePath), pluginSubdirectory)
|
|
245
|
+
}
|
|
246
|
+
}
|
|
247
|
+
|
|
248
|
+
export class DirectorySnapshotLoader implements SnapshotLoader {
|
|
249
|
+
async load(
|
|
250
|
+
_repositoryUrl: string,
|
|
251
|
+
_commitSha: string,
|
|
252
|
+
pluginSubdirectory: string
|
|
253
|
+
): Promise<Snapshot> {
|
|
254
|
+
throw new Error(`DIRECTORY_LOADER_REQUIRES_ROOT: ${pluginSubdirectory}`)
|
|
255
|
+
}
|
|
256
|
+
}
|
|
257
|
+
|
|
258
|
+
export async function snapshotFromDirectory(
|
|
259
|
+
root: string,
|
|
260
|
+
pluginSubdirectory: string
|
|
261
|
+
): Promise<Snapshot> {
|
|
262
|
+
const directory = resolve(root, pluginSubdirectory)
|
|
263
|
+
const files = new Map<string, Uint8Array>()
|
|
264
|
+
const symlinks: string[] = []
|
|
265
|
+
const rootStat = await fs.lstat(directory)
|
|
266
|
+
if (rootStat.isSymbolicLink()) return { files, symlinks: [pluginSubdirectory || '.'] }
|
|
267
|
+
await walk(directory, directory, files, symlinks)
|
|
268
|
+
return { files, symlinks }
|
|
269
|
+
}
|
|
270
|
+
|
|
271
|
+
async function walk(
|
|
272
|
+
root: string,
|
|
273
|
+
current: string,
|
|
274
|
+
files: Map<string, Uint8Array>,
|
|
275
|
+
symlinks: string[]
|
|
276
|
+
): Promise<void> {
|
|
277
|
+
for (const entry of await fs.readdir(current, { withFileTypes: true })) {
|
|
278
|
+
const absolute = join(current, entry.name)
|
|
279
|
+
const relativePath = safeRelativePath(relative(root, absolute), 'DIRECTORY_PATH_INVALID')
|
|
280
|
+
if (entry.isSymbolicLink()) {
|
|
281
|
+
symlinks.push(relativePath)
|
|
282
|
+
continue
|
|
283
|
+
}
|
|
284
|
+
if (entry.isDirectory()) await walk(root, absolute, files, symlinks)
|
|
285
|
+
else if (entry.isFile()) files.set(relativePath, await fs.readFile(absolute))
|
|
286
|
+
}
|
|
287
|
+
}
|
|
@@ -0,0 +1,84 @@
|
|
|
1
|
+
import { copyFile } from 'node:fs/promises'
|
|
2
|
+
import { join } from 'node:path'
|
|
3
|
+
import { expect, test } from 'bun:test'
|
|
4
|
+
import { synchronizePortable } from './portable-catalog.js'
|
|
5
|
+
import { withPortableFixture } from '../test-support/portable-fixture.js'
|
|
6
|
+
|
|
7
|
+
test('rebuilds and reports source metadata changes when source commits stay fixed', () =>
|
|
8
|
+
withPortableFixture(async (input, root) => {
|
|
9
|
+
const first = await synchronizePortable(input)
|
|
10
|
+
const old = input.sources[0]!
|
|
11
|
+
await copyFile(
|
|
12
|
+
join(root, 'fixtures/source/marketplace.json'),
|
|
13
|
+
join(root, 'fixtures/source/alternate.json')
|
|
14
|
+
)
|
|
15
|
+
const cases = [
|
|
16
|
+
{ ...old, displayName: 'Renamed source' },
|
|
17
|
+
{ ...old, defaultBranch: 'stable' },
|
|
18
|
+
{ ...old, manifestPath: 'alternate.json' },
|
|
19
|
+
{ ...old, marketplaceDialect: 'openai' as const },
|
|
20
|
+
]
|
|
21
|
+
for (const source of cases) {
|
|
22
|
+
const result = await synchronizePortable({
|
|
23
|
+
...input,
|
|
24
|
+
sources: [source],
|
|
25
|
+
existingLock: first.lock!,
|
|
26
|
+
existingCatalog: first.catalog!,
|
|
27
|
+
})
|
|
28
|
+
expect(result.changed).toBe(true)
|
|
29
|
+
expect(result.changeReport.changedSources).toContain(source.sourceId)
|
|
30
|
+
expect(result.catalog?.sources[0]?.displayName).toBe(source.displayName)
|
|
31
|
+
expect(result.lock?.sources[0]?.defaultBranch).toBe(source.defaultBranch)
|
|
32
|
+
expect(result.lock?.sources[0]?.manifestPath).toBe(source.manifestPath)
|
|
33
|
+
expect(result.lock?.sources[0]?.marketplaceDialect).toBe(source.marketplaceDialect)
|
|
34
|
+
const repeated = await synchronizePortable({
|
|
35
|
+
...input,
|
|
36
|
+
sources: [source],
|
|
37
|
+
existingLock: result.lock!,
|
|
38
|
+
existingCatalog: result.catalog!,
|
|
39
|
+
})
|
|
40
|
+
expect(repeated.changed).toBe(false)
|
|
41
|
+
}
|
|
42
|
+
const equivalent = await synchronizePortable({
|
|
43
|
+
...input,
|
|
44
|
+
sources: [{ ...old, repositoryUrl: `${old.repositoryUrl}.git` }],
|
|
45
|
+
existingLock: first.lock!,
|
|
46
|
+
existingCatalog: first.catalog!,
|
|
47
|
+
})
|
|
48
|
+
expect(equivalent.changed).toBe(false)
|
|
49
|
+
}))
|
|
50
|
+
|
|
51
|
+
test('rebuilds if the marketplace repository changes while external plugin pins stay equal', () =>
|
|
52
|
+
withPortableFixture(async (input, root) => {
|
|
53
|
+
const { writeFile } = await import('node:fs/promises')
|
|
54
|
+
await writeFile(
|
|
55
|
+
join(root, 'fixtures/source/marketplace.json'),
|
|
56
|
+
JSON.stringify({
|
|
57
|
+
plugins: [
|
|
58
|
+
{ name: 'remote', source: { source: 'git', url: 'https://github.com/example/external' } },
|
|
59
|
+
],
|
|
60
|
+
})
|
|
61
|
+
)
|
|
62
|
+
const snapshotLoader = {
|
|
63
|
+
load: async () => ({
|
|
64
|
+
files: new Map([['plugin.json', new TextEncoder().encode('{"name":"Remote"}')]]),
|
|
65
|
+
symlinks: [],
|
|
66
|
+
}),
|
|
67
|
+
}
|
|
68
|
+
const options = { ...input, snapshotLoader }
|
|
69
|
+
const first = await synchronizePortable(options)
|
|
70
|
+
const source = {
|
|
71
|
+
...input.sources[0]!,
|
|
72
|
+
repositoryUrl: 'https://github.com/example/moved-marketplace',
|
|
73
|
+
}
|
|
74
|
+
const result = await synchronizePortable({
|
|
75
|
+
...options,
|
|
76
|
+
sources: [source],
|
|
77
|
+
existingLock: first.lock!,
|
|
78
|
+
})
|
|
79
|
+
expect(result.changed).toBe(true)
|
|
80
|
+
expect(result.lock?.sources[0]?.repositoryUrl).toBe(source.repositoryUrl)
|
|
81
|
+
expect(result.catalog?.sources[0]?.repositoryUrl).toBe(source.repositoryUrl)
|
|
82
|
+
expect(result.changeReport.changedSources).toEqual([source.sourceId])
|
|
83
|
+
expect(result.lock?.sources[0]?.pluginPins).toEqual(first.lock?.sources[0]?.pluginPins)
|
|
84
|
+
}))
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { mkdtemp, rm, writeFile } from 'node:fs/promises'
|
|
1
|
+
import { mkdir, mkdtemp, rm, writeFile } from 'node:fs/promises'
|
|
2
2
|
import { join } from 'node:path'
|
|
3
3
|
import { spawnSync } from 'node:child_process'
|
|
4
4
|
import { pathToFileURL } from 'node:url'
|
|
@@ -11,20 +11,17 @@ test('source digests agree across consumer locales and equivalent Unicode path o
|
|
|
11
11
|
try {
|
|
12
12
|
const entry = join(directory, 'entry.ts')
|
|
13
13
|
const output = join(directory, 'consumer.mjs')
|
|
14
|
-
await writeFile(
|
|
15
|
-
entry,
|
|
16
|
-
`export { bytesDigest } from '../index.ts'\nexport { sourceDigest } from '../agent-plugins-catalog.ts'\n`
|
|
17
|
-
)
|
|
14
|
+
await writeFile(entry, `export { sourceDigest } from '../digest.ts'\n`)
|
|
18
15
|
const build = await Bun.build({ entrypoints: [entry], target: 'node', packages: 'external' })
|
|
19
16
|
expect(build.success).toBe(true)
|
|
20
17
|
await Bun.write(output, build.outputs[0]!)
|
|
21
18
|
const script = `
|
|
22
|
-
import {
|
|
19
|
+
import { sourceDigest } from ${JSON.stringify(pathToFileURL(output).href)};
|
|
23
20
|
const paths = ['z.txt', 'ä.txt', 'é.txt', 'e\\u0301.txt'];
|
|
24
21
|
const entries = paths.map((path) => [path, new TextEncoder().encode(path)]);
|
|
25
22
|
const forward = new Map(entries), reverse = new Map([...entries].reverse());
|
|
26
23
|
console.log(JSON.stringify([
|
|
27
|
-
|
|
24
|
+
sourceDigest(forward), sourceDigest(reverse)
|
|
28
25
|
]));
|
|
29
26
|
`
|
|
30
27
|
const results = ['en_US.UTF-8', 'sv_SE.UTF-8'].map((locale) => {
|
|
@@ -34,7 +31,6 @@ test('source digests agree across consumer locales and equivalent Unicode path o
|
|
|
34
31
|
})
|
|
35
32
|
if (result.error) throw result.error
|
|
36
33
|
if (result.status !== 0) throw new Error(result.stderr)
|
|
37
|
-
expect(result.status).toBe(0)
|
|
38
34
|
return JSON.parse(result.stdout) as string[]
|
|
39
35
|
})
|
|
40
36
|
expect(new Set(results.flat()).size).toBe(1)
|
|
@@ -42,3 +38,60 @@ test('source digests agree across consumer locales and equivalent Unicode path o
|
|
|
42
38
|
await rm(directory, { recursive: true, force: true })
|
|
43
39
|
}
|
|
44
40
|
})
|
|
41
|
+
|
|
42
|
+
test('source lock identity is independent of the consumer locale', async () => {
|
|
43
|
+
const directory = await mkdtemp(join(import.meta.dirname, '.source-lock-'))
|
|
44
|
+
try {
|
|
45
|
+
const source = join(directory, 'fixture', 'source')
|
|
46
|
+
await mkdir(source, { recursive: true })
|
|
47
|
+
await writeFile(
|
|
48
|
+
join(source, 'marketplace.json'),
|
|
49
|
+
JSON.stringify({
|
|
50
|
+
plugins: [
|
|
51
|
+
{ name: 'a-plugin', source: './a' },
|
|
52
|
+
{ name: 'ä-plugin', source: './u' },
|
|
53
|
+
{ name: 'z-plugin', source: './z' },
|
|
54
|
+
],
|
|
55
|
+
})
|
|
56
|
+
)
|
|
57
|
+
const entry = join(directory, 'entry.ts')
|
|
58
|
+
const output = join(directory, 'consumer.mjs')
|
|
59
|
+
await writeFile(entry, "export { synchronize } from '../index.ts'\n")
|
|
60
|
+
const build = await Bun.build({ entrypoints: [entry], target: 'node', packages: 'external' })
|
|
61
|
+
expect(build.success).toBe(true)
|
|
62
|
+
await Bun.write(output, build.outputs[0]!)
|
|
63
|
+
const script = `
|
|
64
|
+
import { synchronize } from ${JSON.stringify(pathToFileURL(output).href)};
|
|
65
|
+
const result = await synchronize({
|
|
66
|
+
mode: 'offline', fixtureRoot: ${JSON.stringify(join(directory, 'fixture'))},
|
|
67
|
+
sources: [{sourceId:'test-source',displayName:'Test',marketplaceDialect:'claude',
|
|
68
|
+
repositoryUrl:'https://github.com/example/plugins',manifestPath:'marketplace.json',
|
|
69
|
+
defaultBranch:'main',trustClassification:'official',fixturePath:'source',
|
|
70
|
+
fixtureCommitSha:'a'.repeat(40)}],
|
|
71
|
+
categoryMap:{aliases:{},fallback:'other'},productAliases:{aliases:{}},
|
|
72
|
+
policy:{allowedRepositoryProtocols:['https:'],allowedRepositoryHosts:['github.com'],
|
|
73
|
+
maxFilesPerPlugin:100,maxBytesPerPlugin:100000,maxFileBytes:20000,
|
|
74
|
+
maxMarketplacePlugins:100,denyExecutableLifecycleScripts:true,
|
|
75
|
+
publishRequiresCompleteContent:false,sensitiveCapabilityTypes:[]},metadataOnly:true
|
|
76
|
+
});
|
|
77
|
+
console.log(JSON.stringify({lockId:result.lock.lockId,pins:result.lock.sources[0].pluginPins}));
|
|
78
|
+
`
|
|
79
|
+
const results = ['en_US.UTF-8', 'sv_SE.UTF-8'].map((locale) => {
|
|
80
|
+
const result = spawnSync('node', ['--input-type=module', '-e', script], {
|
|
81
|
+
encoding: 'utf8',
|
|
82
|
+
env: { ...process.env, LANG: locale, LC_ALL: locale },
|
|
83
|
+
})
|
|
84
|
+
if (result.error) throw result.error
|
|
85
|
+
if (result.status !== 0) throw new Error(result.stderr)
|
|
86
|
+
return JSON.parse(result.stdout) as { lockId: string; pins: { pluginName: string }[] }
|
|
87
|
+
})
|
|
88
|
+
expect(results[0]).toEqual(results[1])
|
|
89
|
+
expect(results[0]!.pins.map((pin) => pin.pluginName)).toEqual([
|
|
90
|
+
'a-plugin',
|
|
91
|
+
'z-plugin',
|
|
92
|
+
'ä-plugin',
|
|
93
|
+
])
|
|
94
|
+
} finally {
|
|
95
|
+
await rm(directory, { recursive: true, force: true })
|
|
96
|
+
}
|
|
97
|
+
})
|