@owlmeans/llm-reader-google-drive 0.0.0-stage → 0.2.1-alpha.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (90) hide show
  1. package/README.md +113 -2
  2. package/agent-meta/manifest.json +16 -0
  3. package/agent-meta/skills/llm-reader-google-drive/SKILL.md +131 -0
  4. package/build/consts.d.ts +45 -0
  5. package/build/consts.d.ts.map +1 -0
  6. package/build/consts.js +54 -0
  7. package/build/consts.js.map +1 -0
  8. package/build/consts.local.d.ts +3 -0
  9. package/build/consts.local.d.ts.map +1 -0
  10. package/build/consts.local.js +3 -0
  11. package/build/consts.local.js.map +1 -0
  12. package/build/errors.d.ts +14 -0
  13. package/build/errors.d.ts.map +1 -0
  14. package/build/errors.js +26 -0
  15. package/build/errors.js.map +1 -0
  16. package/build/index.d.ts +6 -0
  17. package/build/index.d.ts.map +1 -0
  18. package/build/index.js +5 -0
  19. package/build/index.js.map +1 -0
  20. package/build/source/api.d.ts +9 -0
  21. package/build/source/api.d.ts.map +1 -0
  22. package/build/source/api.js +80 -0
  23. package/build/source/api.js.map +1 -0
  24. package/build/source/consts.local.d.ts +33 -0
  25. package/build/source/consts.local.d.ts.map +1 -0
  26. package/build/source/consts.local.js +39 -0
  27. package/build/source/consts.local.js.map +1 -0
  28. package/build/source/embedded.d.ts +10 -0
  29. package/build/source/embedded.d.ts.map +1 -0
  30. package/build/source/embedded.js +82 -0
  31. package/build/source/embedded.js.map +1 -0
  32. package/build/source/http.d.ts +6 -0
  33. package/build/source/http.d.ts.map +1 -0
  34. package/build/source/http.js +61 -0
  35. package/build/source/http.js.map +1 -0
  36. package/build/source/probe.d.ts +10 -0
  37. package/build/source/probe.d.ts.map +1 -0
  38. package/build/source/probe.js +80 -0
  39. package/build/source/probe.js.map +1 -0
  40. package/build/source/types.local.d.ts +87 -0
  41. package/build/source/types.local.d.ts.map +1 -0
  42. package/build/source/types.local.js +2 -0
  43. package/build/source/types.local.js.map +1 -0
  44. package/build/source.d.ts +22 -0
  45. package/build/source.d.ts.map +1 -0
  46. package/build/source.js +169 -0
  47. package/build/source.js.map +1 -0
  48. package/build/types.d.ts +79 -0
  49. package/build/types.d.ts.map +1 -0
  50. package/build/types.js +2 -0
  51. package/build/types.js.map +1 -0
  52. package/build/url/consts.local.d.ts +10 -0
  53. package/build/url/consts.local.d.ts.map +1 -0
  54. package/build/url/consts.local.js +15 -0
  55. package/build/url/consts.local.js.map +1 -0
  56. package/build/url.d.ts +8 -0
  57. package/build/url.d.ts.map +1 -0
  58. package/build/url.js +117 -0
  59. package/build/url.js.map +1 -0
  60. package/build/utils/names/types.d.ts +10 -0
  61. package/build/utils/names/types.d.ts.map +1 -0
  62. package/build/utils/names/types.js +2 -0
  63. package/build/utils/names/types.js.map +1 -0
  64. package/build/utils/names.d.ts +4 -0
  65. package/build/utils/names.d.ts.map +1 -0
  66. package/build/utils/names.js +31 -0
  67. package/build/utils/names.js.map +1 -0
  68. package/package.json +62 -4
  69. package/src/consts.local.ts +2 -0
  70. package/src/consts.ts +60 -0
  71. package/src/errors.ts +30 -0
  72. package/src/index.ts +7 -0
  73. package/src/source/api.ts +81 -0
  74. package/src/source/consts.local.ts +52 -0
  75. package/src/source/embedded.ts +83 -0
  76. package/src/source/http.ts +66 -0
  77. package/src/source/probe.ts +84 -0
  78. package/src/source/types.local.ts +91 -0
  79. package/src/source.ts +179 -0
  80. package/src/types.ts +86 -0
  81. package/src/url/consts.local.ts +19 -0
  82. package/src/url.ts +121 -0
  83. package/src/utils/names/types.ts +9 -0
  84. package/src/utils/names.ts +36 -0
  85. package/tests/context.ts +212 -0
  86. package/tests/fixtures.ts +88 -0
  87. package/tests/live.spec.ts +53 -0
  88. package/tests/source.spec.ts +189 -0
  89. package/tests/tsconfig.json +12 -0
  90. package/tsconfig.json +16 -0
@@ -0,0 +1,212 @@
1
+ import { DEFAULT_COLLECT_LIMITS } from '@owlmeans/llm-reader'
2
+ import type { ReaderSourceContext } from '@owlmeans/llm-reader'
3
+ import { makeGates } from '@owlmeans/test'
4
+ import type { GoogleDriveOrigins } from '../src/index.js'
5
+ import { bytesOf, childrenOf, itemOf, ROOT } from './fixtures.js'
6
+ import type { StandInItem } from './fixtures.js'
7
+
8
+ export const gates = makeGates({ live: ['LLM_READER_GDRIVE_LIVE'] })
9
+
10
+ export const API_KEY = 'test-api-key'
11
+ export const WARN_UUID = 'b1a2c3d4-uuid'
12
+ /** The size the Drive API declares for the file behind the warning form — past Drive's 100 MiB scan limit. */
13
+ export const WARN_DECLARED_SIZE = 150 * 1024 * 1024
14
+
15
+ export interface StandInHit {
16
+ role: 'drive' | 'usercontent' | 'docs' | 'accounts' | 'api' | 'short'
17
+ /** Path and query as requested. */
18
+ url: string
19
+ headers: Record<string, string>
20
+ }
21
+
22
+ export interface DriveStandIn {
23
+ /** Origins pointing a source at the stand-in. */
24
+ origins: Required<GoogleDriveOrigins>
25
+ /** A link shortener that redirects to the root folder's share link. */
26
+ short: string
27
+ hits: StandInHit[]
28
+ stop: () => void
29
+ }
30
+
31
+ const escapeHtml = (text: string): string => text.replace(/&/g, '&amp;').replace(/</g, '&lt;').replace(/>/g, '&gt;').replace(/"/g, '&quot;')
32
+
33
+ const html = (body: string, status = 200, headers: Record<string, string> = {}): Response =>
34
+ new Response(body, { status, headers: { 'content-type': 'text/html; charset=utf-8', ...headers } })
35
+
36
+ const notFound = (): Response => html('<html><title>Error 404 (Not Found)!!1</title><p>404. That’s an error.</p></html>', 404)
37
+
38
+ const GOOGLE_TYPES: Record<StandInItem['kind'], string | undefined> = {
39
+ folder: 'application/vnd.google-apps.folder',
40
+ document: 'application/vnd.google-apps.document',
41
+ form: 'application/vnd.google-apps.form',
42
+ file: undefined,
43
+ }
44
+
45
+ /**
46
+ * Real local HTTP servers answering like Google Drive: the embedded folder view (Drive's own markup),
47
+ * `open?id=` redirects, usercontent downloads with Range, Content-Disposition, the "can't scan for
48
+ * viruses" form and the quota page, Docs exports behind a redirect, the sign-in host private items
49
+ * redirect to, and Drive API v3 (`X-Goog-Api-Key`, pages of two, a refused key) — plus a shortener.
50
+ */
51
+ export const startDriveStandIn = (): DriveStandIn => {
52
+ const hits: StandInHit[] = []
53
+ const servers: Array<ReturnType<typeof Bun.serve>> = []
54
+ const origins = { drive: '', usercontent: '', docs: '', accounts: '', api: '' }
55
+
56
+ const serve = (role: StandInHit['role'], answer: (url: URL, request: Request) => Response | Promise<Response>): string => {
57
+ const server = Bun.serve({
58
+ port: 0,
59
+ hostname: '127.0.0.1',
60
+ fetch: request => {
61
+ const url = new URL(request.url)
62
+ hits.push({ role, url: `${url.pathname}${url.search}`, headers: Object.fromEntries(request.headers.entries()) })
63
+ return answer(url, request)
64
+ },
65
+ })
66
+ servers.push(server)
67
+ return `http://127.0.0.1:${server.port}`
68
+ }
69
+
70
+ const signIn = (): Response => new Response(null, { status: 302, headers: { location: `${origins.accounts}/v3/signin/identifier?continue=drive` } })
71
+
72
+ const keyed = (item: StandInItem, url: URL): boolean => item.resourceKey == null || url.searchParams.get('resourcekey') === item.resourceKey
73
+
74
+ const hrefOf = (item: StandInItem): string => {
75
+ switch (item.kind) {
76
+ case 'folder': return `${origins.drive}/drive/folders/${item.id}${item.resourceKey != null ? `?resourcekey=${item.resourceKey}` : ''}`
77
+ case 'document': return `${origins.docs}/document/d/${item.id}/edit?usp=drive_web`
78
+ case 'form': return `${origins.docs}/forms/d/${item.id}/edit?usp=drive_web`
79
+ default: return `${origins.drive}/file/d/${item.id}/view?usp=drive_web`
80
+ }
81
+ }
82
+
83
+ const entryOf = (item: StandInItem): string => {
84
+ const type = GOOGLE_TYPES[item.kind] ?? item.mime ?? 'application/octet-stream'
85
+ return `<div class="flip-entry" id="entry-${item.id}" tabindex="0" role="link"><div class="flip-entry-info"><a href="${escapeHtml(hrefOf(item))}" target="_blank">`
86
+ + `<div class="flip-entry-visual"><div class="flip-entry-visual-card"><div class="flip-entry-icon"><img src="https://drive-thirdparty.googleusercontent.com/128/type/${type}" alt=""/></div></div></div>`
87
+ + `<div class="flip-entry-list-icon"><img src="https://drive-thirdparty.googleusercontent.com/16/type/${type}" alt=""/></div>`
88
+ + `<div class="flip-entry-title">${escapeHtml(item.name)}</div></a></div><div class="flip-entry-last-modified"><div>Nov 11</div></div></div>`
89
+ }
90
+
91
+ const folderView = (folder: StandInItem): string =>
92
+ `<!DOCTYPE html><html><head><title>${escapeHtml(folder.name)}</title><meta http-equiv="content-type" content="text/html; charset=utf-8"/></head>`
93
+ + `<body class="flip-embedded"><div class="flip-butter-container"><div id="flip-butter"></div></div><div id="flip-contents" class="flip-list-view">`
94
+ + `<div class="flip-entries">${childrenOf(folder.id).map(entryOf).join('')}</div></div></body></html>`
95
+
96
+ origins.drive = serve('drive', url => {
97
+ const id = url.searchParams.get('id') ?? ''
98
+ if (url.pathname === '/embeddedfolderview') {
99
+ const folder = itemOf(id)
100
+ if (folder?.kind !== 'folder' || !keyed(folder, url)) return notFound()
101
+ if (folder.private === true) return signIn()
102
+ if (folder.stall === true) return new Promise(resolve => setTimeout(() => resolve(html(folderView(folder))), 10_000))
103
+ return html(folderView(folder))
104
+ }
105
+ if (url.pathname === '/open') {
106
+ const item = itemOf(id)
107
+ if (item == null) return notFound()
108
+ const location = item.kind === 'folder' ? `/drive/folders/${id}?usp=drive_open`
109
+ : item.kind === 'document' ? `${origins.docs}/document/d/${id}/edit?usp=drive_open` : `/file/d/${id}/view?usp=drive_open`
110
+ return new Response(null, { status: 307, headers: { location, 'content-type': 'application/binary' } })
111
+ }
112
+ if (/^\/(drive\/folders|file\/d)\//.test(url.pathname)) return html('<!DOCTYPE html><html><head><title>Drive</title></head><body><script>app()</script></body></html>')
113
+ return notFound()
114
+ })
115
+
116
+ origins.usercontent = serve('usercontent', (url, request) => {
117
+ const item = itemOf(url.searchParams.get('id') ?? '')
118
+ if (url.pathname !== '/download' || item?.kind !== 'file' || !keyed(item, url)) return notFound()
119
+ if (item.private === true) return signIn()
120
+ if (item.quota === true) {
121
+ return html('<html><head><title>Google Drive - Quota exceeded</title></head><body><p>Sorry, you can\'t view or download this file at this time.</p>'
122
+ + '<p>Too many users have viewed or downloaded this file recently. Please try accessing the file again later.</p></body></html>')
123
+ }
124
+ if (item.warn === true && url.searchParams.get('uuid') !== WARN_UUID) {
125
+ return html(`<!DOCTYPE html><html><head><title>Google Drive - Virus scan warning</title></head><body><div class="uc-main"><div id="uc-text">`
126
+ + `<p class="uc-warning-caption">Google Drive can't scan this file for viruses.</p><p class="uc-warning-subcaption"><span class="uc-name-size"><a href="/open?id=${item.id}">${escapeHtml(item.name)}</a> (64K)</span> is too large for Google to scan for viruses. Would you still like to download this file?</p>`
127
+ + `<form id="download-form" action="${origins.usercontent}/download" method="get"><input type="submit" id="uc-download-link" class="goog-inline-block jfk-button jfk-button-action" value="Download anyway"/>`
128
+ + `<input type="hidden" name="id" value="${item.id}"><input type="hidden" name="export" value="download"><input type="hidden" name="confirm" value="t"><input type="hidden" name="uuid" value="${WARN_UUID}"></form></div></div></body></html>`)
129
+ }
130
+ const bytes = bytesOf(item)
131
+ const ascii = /^[\x20-\x7e]*$/.test(item.name)
132
+ const headers: Record<string, string> = {
133
+ 'content-type': 'application/octet-stream',
134
+ 'content-disposition': ascii ? `attachment; filename="${item.name}"` : `attachment; filename="${item.name.replace(/[^\x20-\x7e]/g, '_')}"; filename*=UTF-8''${encodeURIComponent(item.name)}`,
135
+ }
136
+ const range = /^bytes=(\d+)-(\d*)$/.exec(request.headers.get('range') ?? '')
137
+ if (range != null) {
138
+ const start = Number(range[1])
139
+ const end = Math.min(range[2] !== '' ? Number(range[2]) : bytes.length - 1, bytes.length - 1)
140
+ return new Response(bytes.slice(start, end + 1), { status: 206, headers: { ...headers, 'content-range': `bytes ${start}-${end}/${bytes.length}` } })
141
+ }
142
+ return new Response(bytes, { status: 200, headers })
143
+ })
144
+
145
+ origins.docs = serve('docs', url => {
146
+ const exported = /^\/exported\/([^/]+)\/([a-z]+)$/.exec(url.pathname)
147
+ if (exported != null) {
148
+ const item = itemOf(exported[1]!)
149
+ const format = item?.exports?.[exported[2]!]
150
+ if (item == null || format == null) return notFound()
151
+ return new Response(format.body, {
152
+ status: 200,
153
+ headers: { 'content-type': format.mime, 'content-disposition': `attachment; filename="${item.name}.${exported[2]}"; filename*=UTF-8''${encodeURIComponent(item.name)}.${exported[2]}` },
154
+ })
155
+ }
156
+ const doc = /^\/document\/d\/([^/]+)\/(export|edit)$/.exec(url.pathname)
157
+ const item = doc != null ? itemOf(doc[1]!) : undefined
158
+ if (doc == null || item?.kind !== 'document') return notFound()
159
+ if (item.private === true) return signIn()
160
+ if (doc[2] === 'edit') return html('<!DOCTYPE html><html><head><title>Spec - Google Docs</title></head><body><script>docs()</script></body></html>')
161
+ // Docs answers an export with a redirect to a googleusercontent host that serves the bytes.
162
+ return new Response(null, { status: 307, headers: { location: `/exported/${item.id}/${url.searchParams.get('format') ?? 'docx'}` } })
163
+ })
164
+
165
+ origins.accounts = serve('accounts', () => html('<!DOCTYPE html><html><head><title>Sign in - Google Accounts</title></head><body>Sign in to continue to Google Drive</body></html>'))
166
+
167
+ const apiError = (status: number, reason: string, message: string, details?: string): Response =>
168
+ new Response(JSON.stringify({
169
+ error: { code: status, message, errors: [{ message, domain: 'global', reason }], ...(details != null ? { status: 'INVALID_ARGUMENT', details: [{ reason: details }] } : {}) },
170
+ }), { status, headers: { 'content-type': 'application/json; charset=UTF-8' } })
171
+
172
+ const apiFile = (item: StandInItem): Record<string, unknown> => ({
173
+ id: item.id, name: item.name, mimeType: GOOGLE_TYPES[item.kind] ?? item.mime ?? 'application/octet-stream',
174
+ // Drive shows its warning form only past its scan limit — the API declares such a size.
175
+ ...(item.kind === 'file' ? { size: String(item.warn === true ? WARN_DECLARED_SIZE : bytesOf(item).length) } : {}),
176
+ ...(item.resourceKey != null ? { resourceKey: item.resourceKey } : {}),
177
+ })
178
+
179
+ origins.api = serve('api', (url, request) => {
180
+ const key = request.headers.get('x-goog-api-key')
181
+ if (key !== API_KEY) return apiError(400, 'badRequest', 'API key not valid. Please pass a valid API key.', 'API_KEY_INVALID')
182
+ if (url.pathname === '/drive/v3/files') {
183
+ const parent = /^'([^']+)' in parents and trashed=false$/.exec(url.searchParams.get('q') ?? '')?.[1]
184
+ const folder = parent != null ? itemOf(parent) : undefined
185
+ if (folder?.kind !== 'folder' || folder.private === true) return apiError(404, 'notFound', `File not found: ${parent}.`)
186
+ const children = childrenOf(folder.id).filter(item => item.private !== true)
187
+ const start = Number(url.searchParams.get('pageToken') ?? '0')
188
+ const page = children.slice(start, start + 2)
189
+ return Response.json({
190
+ files: page.map(apiFile),
191
+ ...(start + 2 < children.length ? { nextPageToken: String(start + 2) } : {}),
192
+ })
193
+ }
194
+ const file = /^\/drive\/v3\/files\/([^/]+)$/.exec(url.pathname)
195
+ const item = file != null ? itemOf(decodeURIComponent(file[1]!)) : undefined
196
+ if (item == null || item.private === true) return apiError(404, 'notFound', 'File not found.')
197
+ return Response.json(apiFile(item))
198
+ })
199
+
200
+ const short = serve('short', () => new Response(null, { status: 302, headers: { location: `${origins.drive}/drive/folders/${ROOT}?usp=sharing` } }))
201
+
202
+ return {
203
+ origins, short: `${short}/s/project-files`, hits,
204
+ stop: () => servers.forEach(server => void server.stop(true)),
205
+ }
206
+ }
207
+
208
+ /** The source seam over plain `fetch` (redirects followed, `response.url` set) — what `expand` needs, without a reader. */
209
+ export const plainContext = (): ReaderSourceContext => ({
210
+ fetch: (url, init = {}) => fetch(url, { ...(init.headers != null ? { headers: init.headers } : {}), ...(init.signal != null ? { signal: init.signal } : {}) }),
211
+ limits: { ...DEFAULT_COLLECT_LIMITS },
212
+ })
@@ -0,0 +1,88 @@
1
+ /** One item the Drive stand-in serves. */
2
+ export interface StandInItem {
3
+ id: string
4
+ /** The folder it is listed in. */
5
+ parent?: string
6
+ kind: 'folder' | 'file' | 'document' | 'form'
7
+ name: string
8
+ mime?: string
9
+ body?: string | Uint8Array
10
+ /** Doc exports by format. */
11
+ exports?: Record<string, { mime: string, body: string }>
12
+ /** Shared privately: Google answers with its sign-in page. */
13
+ private?: boolean
14
+ /** Download quota exhausted. */
15
+ quota?: boolean
16
+ /** Served only after Drive's "can't scan for viruses" form (its `uuid`). */
17
+ warn?: boolean
18
+ /** The folder view never answers. */
19
+ stall?: boolean
20
+ /** A pre-2021 link: the item is reachable only with its resource key. */
21
+ resourceKey?: string
22
+ }
23
+
24
+ /** Bytes that are not text. */
25
+ const pattern = (size: number): Uint8Array => Uint8Array.from({ length: size }, (_, index) => (index * 31 + 7) % 251)
26
+
27
+ export const PNG = Uint8Array.from([0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a, 0, 0, 0, 13, 0x49, 0x48, 0x44, 0x52, 0, 0, 0, 1, 0, 0, 0, 1, 8, 6, 0, 0, 0])
28
+
29
+ export const ROOT = 'rootFolder0001'
30
+ export const ASSETS = 'subAssets0001'
31
+ export const DEEP = 'subDeep000001'
32
+ export const WIDE = 'wideFolder001'
33
+ export const PRIVATE_FOLDER = 'privateFold01'
34
+ export const STALL_FOLDER = 'stallFolder01'
35
+ export const KEYED_FOLDER = 'keyedFolder01'
36
+ export const KEYED_RESOURCE = '0-keyedResKey'
37
+ export const LOGO = 'fileLogo00001'
38
+ export const BIG = 'fileBig000001'
39
+ export const SPEC = 'docSpec000001'
40
+ export const SECRET = 'fileSecret001'
41
+ export const QUOTA = 'fileQuota0001'
42
+ export const ACCENTED = 'fileAccent001'
43
+ export const MISSING = 'missingItem01'
44
+
45
+ export const BIG_BODY = pattern(64 * 1024)
46
+
47
+ export const ITEMS: StandInItem[] = [
48
+ { id: ROOT, kind: 'folder', name: 'Project' },
49
+ { id: 'fileNotes0001', parent: ROOT, kind: 'file', name: 'notes.md', mime: 'text/markdown', body: '# Notes\n\nPlain words.\n' },
50
+ { id: LOGO, parent: ROOT, kind: 'file', name: 'logo.svg', mime: 'image/svg+xml', body: '<svg xmlns="http://www.w3.org/2000/svg" width="1" height="1"/>' },
51
+ { id: 'fileIndex0001', parent: ROOT, kind: 'file', name: 'index.html', mime: 'text/html', body: '<!doctype html><html><body><script>alert(1)</script>Kept as served</body></html>' },
52
+ { id: BIG, parent: ROOT, kind: 'file', name: 'big.bin', mime: 'application/octet-stream', body: BIG_BODY, warn: true },
53
+ {
54
+ id: SPEC, parent: ROOT, kind: 'document', name: 'Spec',
55
+ exports: {
56
+ docx: { mime: 'application/vnd.openxmlformats-officedocument.wordprocessingml.document', body: 'PK-docx-of-Spec' },
57
+ md: { mime: 'text/markdown', body: '# Spec\n\nExported as Markdown.\n' },
58
+ },
59
+ },
60
+ { id: 'formPoll00001', parent: ROOT, kind: 'form', name: 'Poll' },
61
+ { id: SECRET, parent: ROOT, kind: 'file', name: 'secret.txt', mime: 'text/plain', body: 'secret', private: true },
62
+ { id: ASSETS, parent: ROOT, kind: 'folder', name: 'assets' },
63
+ { id: 'filePhoto0001', parent: ASSETS, kind: 'file', name: 'photo.png', mime: 'image/png', body: PNG },
64
+ { id: DEEP, parent: ASSETS, kind: 'folder', name: 'deep' },
65
+ { id: 'fileDeep00001', parent: DEEP, kind: 'file', name: 'deep.txt', mime: 'text/plain', body: 'deep\n' },
66
+
67
+ { id: WIDE, kind: 'folder', name: 'Wide' },
68
+ ...Array.from({ length: 12 }, (_, index): StandInItem => ({
69
+ id: `fileWide${String(index).padStart(5, '0')}`, parent: WIDE, kind: 'file', name: `wide-${String(index).padStart(2, '0')}.txt`,
70
+ mime: 'text/plain', body: `wide ${index}\n`,
71
+ })),
72
+
73
+ { id: KEYED_FOLDER, kind: 'folder', name: 'Keyed', resourceKey: KEYED_RESOURCE },
74
+ { id: 'fileKeyed0001', parent: KEYED_FOLDER, kind: 'file', name: 'keyed.txt', mime: 'text/plain', body: 'keyed\n' },
75
+
76
+ { id: PRIVATE_FOLDER, kind: 'folder', name: 'Hidden', private: true },
77
+ { id: STALL_FOLDER, kind: 'folder', name: 'Slow', stall: true },
78
+ { id: QUOTA, kind: 'file', name: 'popular.zip', mime: 'application/zip', body: 'zip', quota: true },
79
+ { id: ACCENTED, kind: 'file', name: 'résumé 2026.txt', mime: 'text/plain', body: 'cv\n' },
80
+ ]
81
+
82
+ /** The bytes of an item as served. */
83
+ export const bytesOf = (item: StandInItem): Uint8Array =>
84
+ typeof item.body === 'string' ? new TextEncoder().encode(item.body) : item.body ?? new Uint8Array()
85
+
86
+ export const itemOf = (id: string): StandInItem | undefined => ITEMS.find(item => item.id === id)
87
+
88
+ export const childrenOf = (id: string): StandInItem[] => ITEMS.filter(item => item.parent === id)
@@ -0,0 +1,53 @@
1
+ import { afterAll, describe, expect, test } from 'bun:test'
2
+ import { createHash } from 'node:crypto'
3
+ import { mkdtemp, readdir, readFile, rm } from 'node:fs/promises'
4
+ import { tmpdir } from 'node:os'
5
+ import { join } from 'node:path'
6
+ import { CollectKind, CollectStatus, createDirectoryCollectSink, createLlmReader } from '@owlmeans/llm-reader'
7
+ import { isSkip } from '@owlmeans/test'
8
+ import { createGoogleDriveSource, GOOGLE_DRIVE_SOURCE } from '../src/index.js'
9
+ import { gates } from './context.js'
10
+
11
+ const gate = gates.live
12
+ const it = isSkip(gate) ? test.skip : test
13
+ const skipped = isSkip(gate) ? ` (skipped: ${gate.reason})` : ''
14
+
15
+ const FOLDER = 'https://drive.google.com/drive/folders/1knWJ30Rs2f6R8QEdch2I9WMQ8xU7k7_G?usp=share_link'
16
+ const FILE = 'https://drive.google.com/file/d/14ensN8ZJqqoWdCJjtOB31bMOz39wy1N3/view?usp=drive_web'
17
+ const SHA256: Record<string, string> = {
18
+ 'crmore-benefits.md': '3e5a0af1e1c9b30883135ae4df70d63039e64687666c1227f7c5585cec0dfe2c',
19
+ 'crmore-logo.svg': 'f96de2f591701b9939799d3f3f2867737267b60fbd0088bc60bd49e70c34f72d',
20
+ 'index.html': '64cde9b44deeb37d28f7cbd0a1badee5c34af2c5d53617b45c4753fd489cb158',
21
+ }
22
+
23
+ const scratch = await mkdtemp(join(tmpdir(), 'llm-reader-google-drive-live-'))
24
+ afterAll(async () => {
25
+ await rm(scratch, { recursive: true, force: true })
26
+ })
27
+
28
+ const digests = async (root: string): Promise<Record<string, string>> => {
29
+ const names = (await readdir(root, { recursive: true, withFileTypes: true })).filter(entry => entry.isFile())
30
+ .map(entry => join(entry.parentPath, entry.name).slice(root.length + 1))
31
+ return Object.fromEntries(await Promise.all(names.map(async name =>
32
+ [name, createHash('sha256').update(await readFile(join(root, name))).digest('hex')] as const)))
33
+ }
34
+
35
+ /** The public Drive, keyless, through a real reader with its default fetch — `LLM_READER_GDRIVE_LIVE=1`. */
36
+ describe(`@owlmeans/llm-reader-google-drive — live Google Drive${skipped}`, () => {
37
+ const reader = createLlmReader({ sources: [createGoogleDriveSource()] })
38
+
39
+ it('a shared folder is exactly its three files, byte for byte', async () => {
40
+ const root = join(scratch, 'folder')
41
+ const result = await reader.collect(FOLDER, { sink: createDirectoryCollectSink(root) })
42
+ expect(result).toMatchObject({ kind: CollectKind.Source, source: GOOGLE_DRIVE_SOURCE, files: 3 })
43
+ expect(result.items.every(item => item.status === CollectStatus.Written)).toBe(true)
44
+ expect(await digests(root)).toEqual(SHA256)
45
+ }, 120_000)
46
+
47
+ it('a single shared file is that file, named as Drive names it', async () => {
48
+ const root = join(scratch, 'file')
49
+ const result = await reader.collect(FILE, { sink: createDirectoryCollectSink(root) })
50
+ expect(result).toMatchObject({ kind: CollectKind.Source, entry: 'crmore-logo.svg', files: 1 })
51
+ expect(await digests(root)).toEqual({ 'crmore-logo.svg': SHA256['crmore-logo.svg'] })
52
+ }, 120_000)
53
+ })
@@ -0,0 +1,189 @@
1
+ import { afterAll, describe, expect, test } from 'bun:test'
2
+ import { mkdtemp, readFile, readdir, rm } from 'node:fs/promises'
3
+ import { tmpdir } from 'node:os'
4
+ import { join } from 'node:path'
5
+ import { CollectKind, CollectStatus, createDirectoryCollectSink, createLlmReader } from '@owlmeans/llm-reader'
6
+ import type { CollectSink, LlmReader } from '@owlmeans/llm-reader'
7
+ import {
8
+ createGoogleDriveSource, createGoogleDriveUrlHelper, GOOGLE_DRIVE_SOURCE, GoogleDriveError, GoogleDriveFailure, GoogleDriveKind,
9
+ } from '../src/index.js'
10
+ import type { GoogleDriveSourceOptions } from '../src/index.js'
11
+ import { API_KEY, plainContext, startDriveStandIn, WARN_DECLARED_SIZE, WARN_UUID } from './context.js'
12
+ import {
13
+ ACCENTED, BIG, BIG_BODY, KEYED_FOLDER, KEYED_RESOURCE, LOGO, MISSING, PNG, PRIVATE_FOLDER, QUOTA, ROOT, SECRET, SPEC, STALL_FOLDER, WIDE,
14
+ } from './fixtures.js'
15
+
16
+ const drive = startDriveStandIn()
17
+ const scratch = await mkdtemp(join(tmpdir(), 'llm-reader-google-drive-'))
18
+ afterAll(async () => {
19
+ drive.stop()
20
+ await rm(scratch, { recursive: true, force: true })
21
+ })
22
+
23
+ let dirs = 0
24
+ const target = (): { root: string, sink: CollectSink } => {
25
+ const root = join(scratch, `run-${++dirs}`)
26
+ return { root, sink: createDirectoryCollectSink(root) }
27
+ }
28
+ const files = async (root: string): Promise<string[]> =>
29
+ (await readdir(root, { recursive: true, withFileTypes: true })).filter(entry => entry.isFile())
30
+ .map(entry => join(entry.parentPath, entry.name).slice(root.length + 1)).sort()
31
+
32
+ /** A real reader with the source pointed at the stand-in; the guard allows only the loopback stand-in. */
33
+ const readerWith = (options: GoogleDriveSourceOptions = {}): LlmReader =>
34
+ createLlmReader({ sources: [createGoogleDriveSource({ origins: drive.origins, ...options })], allowHosts: ['127.0.0.1'], timeout: 1500, s3: false })
35
+
36
+ const reader = readerWith()
37
+ const folderLink = (id: string, query = '?usp=share_link'): string => `${drive.origins.drive}/drive/folders/${id}${query}`
38
+ const fileLink = (id: string): string => `${drive.origins.drive}/file/d/${id}/view?usp=drive_web`
39
+
40
+ const ROOT_FILES = ['Spec.docx', 'assets/deep/deep.txt', 'assets/photo.png', 'big.bin', 'index.html', 'logo.svg', 'notes.md']
41
+
42
+ describe('@owlmeans/llm-reader-google-drive', () => {
43
+ test('Drive and Docs links are recognised from the URL alone; URLs are built for downloads, exports and listings', () => {
44
+ const source = createGoogleDriveSource()
45
+ const id = '1knWJ30Rs2f6R8QEdch2I9WMQ8xU7k7_G'
46
+ for (const url of [
47
+ `https://drive.google.com/drive/folders/${id}?usp=share_link`, `https://drive.google.com/drive/u/1/folders/${id}`,
48
+ `https://drive.google.com/drive/folders/${id}?resourcekey=0-abc`, `https://drive.google.com/file/d/${id}/view?usp=drive_web`,
49
+ `https://drive.google.com/open?id=${id}`, `https://drive.google.com/uc?id=${id}&export=download`, `http://drive.google.com/file/d/${id}`,
50
+ `https://drive.usercontent.google.com/download?id=${id}&export=download`, `https://docs.google.com/document/d/${id}/edit`,
51
+ `https://docs.google.com/spreadsheets/d/${id}/edit#gid=0`, `https://docs.google.com/presentation/d/${id}/edit`,
52
+ `https://docs.google.com/drawings/d/${id}/edit`, `https://docs.google.com/document/u/0/d/${id}/edit`,
53
+ ]) expect([url, source.matches(url)]).toEqual([url, true])
54
+ for (const url of [
55
+ 'https://drive.google.com/', 'https://drive.google.com/drive/my-drive', `https://docs.google.com/forms/d/${id}/viewform`,
56
+ `https://docs.google.com/document/d/e/2PACX-1vQx/pub`, 'https://drive.google.com/drive/folders/short', `https://example.com/drive/folders/${id}`,
57
+ `ftp://drive.google.com/file/d/${id}`, 'not a url',
58
+ ]) expect([url, source.matches(url)]).toEqual([url, false])
59
+
60
+ const urls = createGoogleDriveUrlHelper({ exports: { document: 'md', spreadsheet: 'csv' } })
61
+ expect(urls.parse(`https://drive.google.com/drive/folders/${id}?resourcekey=0-abc`)).toEqual({ kind: GoogleDriveKind.Folder, id, resourceKey: '0-abc' })
62
+ expect(urls.parse(`https://drive.google.com/open?id=${id}`)).toEqual({ kind: GoogleDriveKind.Unresolved, id })
63
+ expect(urls.parse(`https://docs.google.com/spreadsheets/d/${id}/edit`)).toEqual({ kind: GoogleDriveKind.Spreadsheet, id })
64
+ expect(urls.downloadUrl(id)).toBe(`https://drive.usercontent.google.com/download?id=${id}&export=download&confirm=t`)
65
+ expect(urls.exportOf(GoogleDriveKind.Document, id)).toEqual({ url: `https://docs.google.com/document/d/${id}/export?format=md`, extension: 'md' })
66
+ expect(urls.exportOf(GoogleDriveKind.Presentation, id).url).toBe(`https://docs.google.com/presentation/d/${id}/export/pptx`)
67
+ expect(urls.folderViewUrl(id, '0-abc')).toBe(`https://drive.google.com/embeddedfolderview?id=${id}&resourcekey=0-abc`)
68
+ })
69
+
70
+ test('a public folder is collected through a real reader: sub-folders, an exported Doc, the warning form followed, private and form items left out', async () => {
71
+ const { root, sink } = target()
72
+ const before = drive.hits.length
73
+ const result = await reader.collect(folderLink(ROOT), { sink, prefix: 'links/01-drive' })
74
+ expect(result).toMatchObject({ kind: CollectKind.Source, source: GOOGLE_DRIVE_SOURCE, files: ROOT_FILES.length })
75
+ expect(result.items.every(item => item.status === CollectStatus.Written)).toBe(true)
76
+ expect(await files(root)).toEqual(ROOT_FILES.map(path => `links/01-drive/${path}`))
77
+ const at = (path: string): Promise<Buffer> => readFile(join(root, 'links/01-drive', path))
78
+ expect(Buffer.compare(await at('big.bin'), BIG_BODY)).toBe(0)
79
+ expect(new Uint8Array(await at('assets/photo.png'))).toEqual(PNG)
80
+ expect((await at('Spec.docx')).toString()).toBe('PK-docx-of-Spec')
81
+ // An HTML file is a file like any other: kept as served, never bundled.
82
+ expect((await at('index.html')).toString()).toContain('<script>alert(1)</script>')
83
+ const items = Object.fromEntries(result.items.map(item => [item.path, item]))
84
+ expect(items['links/01-drive/notes.md']?.mime).toBe('text/markdown')
85
+ expect(items['links/01-drive/big.bin']?.url).toContain(`uuid=${WARN_UUID}`)
86
+
87
+ const hits = drive.hits.slice(before)
88
+ // Probes are ranged; the collector's own downloads are not.
89
+ const probes = hits.filter(hit => hit.role === 'usercontent' && hit.headers.range === 'bytes=0-0')
90
+ expect(probes.length).toBeGreaterThanOrEqual(6)
91
+ expect(hits.filter(hit => hit.role === 'drive' && hit.url.startsWith('/embeddedfolderview')).length).toBe(3)
92
+ expect(hits.some(hit => hit.url.includes(SECRET) && hit.headers.range == null)).toBe(false)
93
+ }, 30_000)
94
+
95
+ test('depth, maxEntries and the collect\'s maxFiles bound the walk; the cut is reported', async () => {
96
+ const shallow = await readerWith({ depth: 0 }).collect(folderLink(ROOT), target())
97
+ expect(shallow.items.map(item => item.path).sort()).toEqual(['Spec.docx', 'big.bin', 'index.html', 'logo.svg', 'notes.md'])
98
+ const one = await readerWith({ depth: 1 }).collect(folderLink(ROOT), target())
99
+ expect(one.items.map(item => item.path)).toContain('assets/photo.png')
100
+ expect(one.items.map(item => item.path)).not.toContain('assets/deep/deep.txt')
101
+
102
+ const limited = await reader.collect(folderLink(WIDE), { ...target(), limits: { maxFiles: 5 } })
103
+ expect(limited.items.filter(item => item.status === CollectStatus.Written)).toHaveLength(5)
104
+ expect(limited.items.filter(item => item.status === CollectStatus.Skipped).map(item => item.reason)).toEqual(['max-files'])
105
+ const capped = await readerWith({ maxEntries: 3 }).collect(folderLink(WIDE), target())
106
+ expect(capped.items.map(item => item.path)).toEqual(['wide-00.txt', 'wide-01.txt', 'wide-02.txt'])
107
+ }, 30_000)
108
+
109
+ test('editor exports follow the options; single links are one entry named as Drive names it', async () => {
110
+ const { root, sink } = target()
111
+ const markdown = await readerWith({ exports: { document: 'md' } }).collect(folderLink(ROOT), { sink, prefix: 'md' })
112
+ expect(markdown.items.map(item => item.path)).toContain('md/Spec.md')
113
+ expect((await readFile(join(root, 'md/Spec.md'), 'utf8'))).toBe('# Spec\n\nExported as Markdown.\n')
114
+
115
+ const single = async (url: string): Promise<{ entry?: string, kind: CollectKind, bytes: number }> => {
116
+ const result = await reader.collect(url, target())
117
+ expect(result.items.every(item => item.status === CollectStatus.Written)).toBe(true)
118
+ return { ...(result.entry != null ? { entry: result.entry } : {}), kind: result.kind, bytes: result.bytes }
119
+ }
120
+ expect(await single(fileLink(LOGO))).toMatchObject({ entry: 'logo.svg', kind: CollectKind.Source })
121
+ expect(await single(`${drive.origins.drive}/uc?id=${LOGO}&export=download`)).toMatchObject({ entry: 'logo.svg' })
122
+ expect(await single(`${drive.origins.drive}/open?id=${LOGO}`)).toMatchObject({ entry: 'logo.svg' })
123
+ expect(await single(fileLink(ACCENTED))).toMatchObject({ entry: 'résumé 2026.txt' })
124
+ expect(await single(fileLink(BIG))).toMatchObject({ entry: 'big.bin', bytes: BIG_BODY.length })
125
+ expect(await single(`${drive.origins.docs}/document/d/${SPEC}/edit?usp=sharing`)).toMatchObject({ entry: 'Spec.docx' })
126
+ const opened = await reader.collect(`${drive.origins.drive}/open?id=${ROOT}`, target())
127
+ expect(opened.files).toBe(ROOT_FILES.length)
128
+ // A link that lands on Drive through a redirect (a shortener) is taken by where it landed.
129
+ const shortened = await reader.collect(drive.short, target())
130
+ expect(shortened).toMatchObject({ kind: CollectKind.Source, source: GOOGLE_DRIVE_SOURCE, files: ROOT_FILES.length })
131
+ // A pre-2021 link carries its resource key to the folder view.
132
+ const keyed = await reader.collect(folderLink(KEYED_FOLDER, `?resourcekey=${KEYED_RESOURCE}`), target())
133
+ expect(keyed.items.map(item => [item.path, item.status])).toEqual([['keyed.txt', CollectStatus.Written]])
134
+ }, 30_000)
135
+
136
+ test('refusals are clear errors: private → not-public, quota, 404 → not-found; a stalled answer times out', async () => {
137
+ const source = createGoogleDriveSource({ origins: drive.origins })
138
+ const reason = async (url: string): Promise<GoogleDriveFailure | string> => {
139
+ const error = await source.expand(url, plainContext()).then(() => 'expanded', (e: unknown) => e)
140
+ return error instanceof GoogleDriveError ? error.reason : String(error)
141
+ }
142
+ expect(await reason(folderLink(PRIVATE_FOLDER))).toBe(GoogleDriveFailure.NotPublic)
143
+ expect(await reason(fileLink(SECRET))).toBe(GoogleDriveFailure.NotPublic)
144
+ expect(await reason(fileLink(QUOTA))).toBe(GoogleDriveFailure.Quota)
145
+ expect(await reason(folderLink(MISSING))).toBe(GoogleDriveFailure.NotFound)
146
+ expect(await reason(folderLink(KEYED_FOLDER))).toBe(GoogleDriveFailure.NotFound)
147
+ expect(await reason('https://example.com/file')).toBe(GoogleDriveFailure.NotALink)
148
+
149
+ const refused = await reader.collect(folderLink(PRIVATE_FOLDER), target())
150
+ expect(refused).toMatchObject({ kind: CollectKind.Source, files: 0 })
151
+ expect(refused.items).toEqual([expect.objectContaining({ status: CollectStatus.Failed, reason: 'llm-reader:fetch:google-drive:not-public' })])
152
+ const started = Date.now()
153
+ const stalled = await reader.collect(folderLink(STALL_FOLDER), target())
154
+ expect(stalled.items[0]).toMatchObject({ status: CollectStatus.Failed, reason: 'llm-reader:fetch:timeout' })
155
+ expect(Date.now() - started).toBeLessThan(5_000)
156
+ }, 30_000)
157
+
158
+ test('with an API key: Drive API pages with sizes and media types, the key only as a header, a refused key is api-key', async () => {
159
+ const before = drive.hits.length
160
+ const source = createGoogleDriveSource({ origins: drive.origins, apiKey: API_KEY })
161
+ const entries = await source.expand(folderLink(ROOT), plainContext())
162
+ // The file behind the warning form declares a size past the default file cap: not probed, admitted by size.
163
+ expect(entries.map(entry => entry.path).sort()).toEqual(ROOT_FILES)
164
+ expect(entries.find(entry => entry.path === 'notes.md')).toMatchObject({ size: 22, mime: 'text/markdown' })
165
+ expect(entries.find(entry => entry.path === 'big.bin')).toMatchObject({ size: WARN_DECLARED_SIZE })
166
+ const apiHits = drive.hits.slice(before).filter(hit => hit.role === 'api')
167
+ expect(apiHits.some(hit => hit.url.includes('pageToken='))).toBe(true)
168
+ expect(apiHits.every(hit => hit.headers['x-goog-api-key'] === API_KEY && !hit.url.includes(API_KEY))).toBe(true)
169
+ expect(drive.hits.slice(before).some(hit => hit.role === 'drive')).toBe(false)
170
+
171
+ const keyed = createLlmReader({ sources: [source], allowHosts: ['127.0.0.1'], timeout: 1500, s3: false })
172
+ const capped = await keyed.collect(folderLink(ROOT), target())
173
+ expect(capped.items.find(item => item.path === 'big.bin')).toMatchObject({ status: CollectStatus.Skipped, reason: 'max-file-bytes' })
174
+ expect(capped.items.filter(item => item.status === CollectStatus.Written)).toHaveLength(ROOT_FILES.length - 1)
175
+ // Past the scan limit and under a raised cap, the file is probed and its warning form followed.
176
+ const { root, sink } = target()
177
+ const raised = await keyed.collect(folderLink(ROOT), { sink, limits: { maxFileBytes: 200 * 1024 * 1024, maxTotalBytes: 400 * 1024 * 1024 } })
178
+ expect(raised.items.find(item => item.path === 'big.bin')).toMatchObject({ status: CollectStatus.Written, bytes: BIG_BODY.length })
179
+ expect(Buffer.compare(await readFile(join(root, 'big.bin')), BIG_BODY)).toBe(0)
180
+ expect((await keyed.collect(`${drive.origins.drive}/open?id=${SPEC}`, target())).entry).toBe('Spec.docx')
181
+
182
+ const wrong = createGoogleDriveSource({ origins: drive.origins, apiKey: 'wrong-key' })
183
+ const error = await wrong.expand(folderLink(ROOT), plainContext()).catch((e: unknown) => e)
184
+ expect(error).toBeInstanceOf(GoogleDriveError)
185
+ expect((error as GoogleDriveError).reason).toBe(GoogleDriveFailure.ApiKey)
186
+ expect(((await source.expand(folderLink(PRIVATE_FOLDER), plainContext()).catch((e: unknown) => e)) as GoogleDriveError).reason)
187
+ .toBe(GoogleDriveFailure.NotFound)
188
+ }, 30_000)
189
+ })
@@ -0,0 +1,12 @@
1
+ {
2
+ "extends": [
3
+ "@owlmeans/dep-config/tsconfig.base.json",
4
+ "@owlmeans/dep-config/tsconfig.node.json"
5
+ ],
6
+ "compilerOptions": {
7
+ "types": ["bun"],
8
+ "rootDir": "../",
9
+ "noEmit": true
10
+ },
11
+ "include": ["./**/*", "../src/**/*"]
12
+ }
package/tsconfig.json ADDED
@@ -0,0 +1,16 @@
1
+ {
2
+ "extends": [
3
+ "@owlmeans/dep-config/tsconfig.base.json",
4
+ "@owlmeans/dep-config/tsconfig.node.json"
5
+ ],
6
+ "compilerOptions": {
7
+ "rootDir": "./src/",
8
+ "outDir": "./build/"
9
+ },
10
+ "exclude": [
11
+ "./dist/**/*",
12
+ "./build/**/*",
13
+ "./tests/**/*",
14
+ "./*.ts"
15
+ ]
16
+ }