@owlmeans/llm-reader-google-drive 0.0.0-stage → 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +113 -2
- package/agent-meta/manifest.json +16 -0
- package/agent-meta/skills/llm-reader-google-drive/SKILL.md +131 -0
- package/build/consts.d.ts +45 -0
- package/build/consts.d.ts.map +1 -0
- package/build/consts.js +54 -0
- package/build/consts.js.map +1 -0
- package/build/consts.local.d.ts +3 -0
- package/build/consts.local.d.ts.map +1 -0
- package/build/consts.local.js +3 -0
- package/build/consts.local.js.map +1 -0
- package/build/errors.d.ts +14 -0
- package/build/errors.d.ts.map +1 -0
- package/build/errors.js +26 -0
- package/build/errors.js.map +1 -0
- package/build/index.d.ts +6 -0
- package/build/index.d.ts.map +1 -0
- package/build/index.js +5 -0
- package/build/index.js.map +1 -0
- package/build/source/api.d.ts +9 -0
- package/build/source/api.d.ts.map +1 -0
- package/build/source/api.js +80 -0
- package/build/source/api.js.map +1 -0
- package/build/source/consts.local.d.ts +33 -0
- package/build/source/consts.local.d.ts.map +1 -0
- package/build/source/consts.local.js +39 -0
- package/build/source/consts.local.js.map +1 -0
- package/build/source/embedded.d.ts +10 -0
- package/build/source/embedded.d.ts.map +1 -0
- package/build/source/embedded.js +82 -0
- package/build/source/embedded.js.map +1 -0
- package/build/source/http.d.ts +6 -0
- package/build/source/http.d.ts.map +1 -0
- package/build/source/http.js +61 -0
- package/build/source/http.js.map +1 -0
- package/build/source/probe.d.ts +10 -0
- package/build/source/probe.d.ts.map +1 -0
- package/build/source/probe.js +80 -0
- package/build/source/probe.js.map +1 -0
- package/build/source/types.local.d.ts +87 -0
- package/build/source/types.local.d.ts.map +1 -0
- package/build/source/types.local.js +2 -0
- package/build/source/types.local.js.map +1 -0
- package/build/source.d.ts +22 -0
- package/build/source.d.ts.map +1 -0
- package/build/source.js +169 -0
- package/build/source.js.map +1 -0
- package/build/types.d.ts +79 -0
- package/build/types.d.ts.map +1 -0
- package/build/types.js +2 -0
- package/build/types.js.map +1 -0
- package/build/url/consts.local.d.ts +10 -0
- package/build/url/consts.local.d.ts.map +1 -0
- package/build/url/consts.local.js +15 -0
- package/build/url/consts.local.js.map +1 -0
- package/build/url.d.ts +8 -0
- package/build/url.d.ts.map +1 -0
- package/build/url.js +117 -0
- package/build/url.js.map +1 -0
- package/build/utils/names/types.d.ts +10 -0
- package/build/utils/names/types.d.ts.map +1 -0
- package/build/utils/names/types.js +2 -0
- package/build/utils/names/types.js.map +1 -0
- package/build/utils/names.d.ts +4 -0
- package/build/utils/names.d.ts.map +1 -0
- package/build/utils/names.js +31 -0
- package/build/utils/names.js.map +1 -0
- package/package.json +62 -4
- package/src/consts.local.ts +2 -0
- package/src/consts.ts +60 -0
- package/src/errors.ts +30 -0
- package/src/index.ts +7 -0
- package/src/source/api.ts +81 -0
- package/src/source/consts.local.ts +52 -0
- package/src/source/embedded.ts +83 -0
- package/src/source/http.ts +66 -0
- package/src/source/probe.ts +84 -0
- package/src/source/types.local.ts +91 -0
- package/src/source.ts +179 -0
- package/src/types.ts +86 -0
- package/src/url/consts.local.ts +19 -0
- package/src/url.ts +121 -0
- package/src/utils/names/types.ts +9 -0
- package/src/utils/names.ts +36 -0
- package/tests/context.ts +212 -0
- package/tests/fixtures.ts +88 -0
- package/tests/live.spec.ts +53 -0
- package/tests/source.spec.ts +189 -0
- package/tests/tsconfig.json +12 -0
- package/tsconfig.json +16 -0
package/tests/context.ts
ADDED
|
@@ -0,0 +1,212 @@
|
|
|
1
|
+
import { DEFAULT_COLLECT_LIMITS } from '@owlmeans/llm-reader'
|
|
2
|
+
import type { ReaderSourceContext } from '@owlmeans/llm-reader'
|
|
3
|
+
import { makeGates } from '@owlmeans/test'
|
|
4
|
+
import type { GoogleDriveOrigins } from '../src/index.js'
|
|
5
|
+
import { bytesOf, childrenOf, itemOf, ROOT } from './fixtures.js'
|
|
6
|
+
import type { StandInItem } from './fixtures.js'
|
|
7
|
+
|
|
8
|
+
export const gates = makeGates({ live: ['LLM_READER_GDRIVE_LIVE'] })
|
|
9
|
+
|
|
10
|
+
export const API_KEY = 'test-api-key'
|
|
11
|
+
export const WARN_UUID = 'b1a2c3d4-uuid'
|
|
12
|
+
/** The size the Drive API declares for the file behind the warning form — past Drive's 100 MiB scan limit. */
|
|
13
|
+
export const WARN_DECLARED_SIZE = 150 * 1024 * 1024
|
|
14
|
+
|
|
15
|
+
export interface StandInHit {
|
|
16
|
+
role: 'drive' | 'usercontent' | 'docs' | 'accounts' | 'api' | 'short'
|
|
17
|
+
/** Path and query as requested. */
|
|
18
|
+
url: string
|
|
19
|
+
headers: Record<string, string>
|
|
20
|
+
}
|
|
21
|
+
|
|
22
|
+
export interface DriveStandIn {
|
|
23
|
+
/** Origins pointing a source at the stand-in. */
|
|
24
|
+
origins: Required<GoogleDriveOrigins>
|
|
25
|
+
/** A link shortener that redirects to the root folder's share link. */
|
|
26
|
+
short: string
|
|
27
|
+
hits: StandInHit[]
|
|
28
|
+
stop: () => void
|
|
29
|
+
}
|
|
30
|
+
|
|
31
|
+
const escapeHtml = (text: string): string => text.replace(/&/g, '&').replace(/</g, '<').replace(/>/g, '>').replace(/"/g, '"')
|
|
32
|
+
|
|
33
|
+
const html = (body: string, status = 200, headers: Record<string, string> = {}): Response =>
|
|
34
|
+
new Response(body, { status, headers: { 'content-type': 'text/html; charset=utf-8', ...headers } })
|
|
35
|
+
|
|
36
|
+
const notFound = (): Response => html('<html><title>Error 404 (Not Found)!!1</title><p>404. That’s an error.</p></html>', 404)
|
|
37
|
+
|
|
38
|
+
const GOOGLE_TYPES: Record<StandInItem['kind'], string | undefined> = {
|
|
39
|
+
folder: 'application/vnd.google-apps.folder',
|
|
40
|
+
document: 'application/vnd.google-apps.document',
|
|
41
|
+
form: 'application/vnd.google-apps.form',
|
|
42
|
+
file: undefined,
|
|
43
|
+
}
|
|
44
|
+
|
|
45
|
+
/**
|
|
46
|
+
* Real local HTTP servers answering like Google Drive: the embedded folder view (Drive's own markup),
|
|
47
|
+
* `open?id=` redirects, usercontent downloads with Range, Content-Disposition, the "can't scan for
|
|
48
|
+
* viruses" form and the quota page, Docs exports behind a redirect, the sign-in host private items
|
|
49
|
+
* redirect to, and Drive API v3 (`X-Goog-Api-Key`, pages of two, a refused key) — plus a shortener.
|
|
50
|
+
*/
|
|
51
|
+
export const startDriveStandIn = (): DriveStandIn => {
|
|
52
|
+
const hits: StandInHit[] = []
|
|
53
|
+
const servers: Array<ReturnType<typeof Bun.serve>> = []
|
|
54
|
+
const origins = { drive: '', usercontent: '', docs: '', accounts: '', api: '' }
|
|
55
|
+
|
|
56
|
+
const serve = (role: StandInHit['role'], answer: (url: URL, request: Request) => Response | Promise<Response>): string => {
|
|
57
|
+
const server = Bun.serve({
|
|
58
|
+
port: 0,
|
|
59
|
+
hostname: '127.0.0.1',
|
|
60
|
+
fetch: request => {
|
|
61
|
+
const url = new URL(request.url)
|
|
62
|
+
hits.push({ role, url: `${url.pathname}${url.search}`, headers: Object.fromEntries(request.headers.entries()) })
|
|
63
|
+
return answer(url, request)
|
|
64
|
+
},
|
|
65
|
+
})
|
|
66
|
+
servers.push(server)
|
|
67
|
+
return `http://127.0.0.1:${server.port}`
|
|
68
|
+
}
|
|
69
|
+
|
|
70
|
+
const signIn = (): Response => new Response(null, { status: 302, headers: { location: `${origins.accounts}/v3/signin/identifier?continue=drive` } })
|
|
71
|
+
|
|
72
|
+
const keyed = (item: StandInItem, url: URL): boolean => item.resourceKey == null || url.searchParams.get('resourcekey') === item.resourceKey
|
|
73
|
+
|
|
74
|
+
const hrefOf = (item: StandInItem): string => {
|
|
75
|
+
switch (item.kind) {
|
|
76
|
+
case 'folder': return `${origins.drive}/drive/folders/${item.id}${item.resourceKey != null ? `?resourcekey=${item.resourceKey}` : ''}`
|
|
77
|
+
case 'document': return `${origins.docs}/document/d/${item.id}/edit?usp=drive_web`
|
|
78
|
+
case 'form': return `${origins.docs}/forms/d/${item.id}/edit?usp=drive_web`
|
|
79
|
+
default: return `${origins.drive}/file/d/${item.id}/view?usp=drive_web`
|
|
80
|
+
}
|
|
81
|
+
}
|
|
82
|
+
|
|
83
|
+
const entryOf = (item: StandInItem): string => {
|
|
84
|
+
const type = GOOGLE_TYPES[item.kind] ?? item.mime ?? 'application/octet-stream'
|
|
85
|
+
return `<div class="flip-entry" id="entry-${item.id}" tabindex="0" role="link"><div class="flip-entry-info"><a href="${escapeHtml(hrefOf(item))}" target="_blank">`
|
|
86
|
+
+ `<div class="flip-entry-visual"><div class="flip-entry-visual-card"><div class="flip-entry-icon"><img src="https://drive-thirdparty.googleusercontent.com/128/type/${type}" alt=""/></div></div></div>`
|
|
87
|
+
+ `<div class="flip-entry-list-icon"><img src="https://drive-thirdparty.googleusercontent.com/16/type/${type}" alt=""/></div>`
|
|
88
|
+
+ `<div class="flip-entry-title">${escapeHtml(item.name)}</div></a></div><div class="flip-entry-last-modified"><div>Nov 11</div></div></div>`
|
|
89
|
+
}
|
|
90
|
+
|
|
91
|
+
const folderView = (folder: StandInItem): string =>
|
|
92
|
+
`<!DOCTYPE html><html><head><title>${escapeHtml(folder.name)}</title><meta http-equiv="content-type" content="text/html; charset=utf-8"/></head>`
|
|
93
|
+
+ `<body class="flip-embedded"><div class="flip-butter-container"><div id="flip-butter"></div></div><div id="flip-contents" class="flip-list-view">`
|
|
94
|
+
+ `<div class="flip-entries">${childrenOf(folder.id).map(entryOf).join('')}</div></div></body></html>`
|
|
95
|
+
|
|
96
|
+
origins.drive = serve('drive', url => {
|
|
97
|
+
const id = url.searchParams.get('id') ?? ''
|
|
98
|
+
if (url.pathname === '/embeddedfolderview') {
|
|
99
|
+
const folder = itemOf(id)
|
|
100
|
+
if (folder?.kind !== 'folder' || !keyed(folder, url)) return notFound()
|
|
101
|
+
if (folder.private === true) return signIn()
|
|
102
|
+
if (folder.stall === true) return new Promise(resolve => setTimeout(() => resolve(html(folderView(folder))), 10_000))
|
|
103
|
+
return html(folderView(folder))
|
|
104
|
+
}
|
|
105
|
+
if (url.pathname === '/open') {
|
|
106
|
+
const item = itemOf(id)
|
|
107
|
+
if (item == null) return notFound()
|
|
108
|
+
const location = item.kind === 'folder' ? `/drive/folders/${id}?usp=drive_open`
|
|
109
|
+
: item.kind === 'document' ? `${origins.docs}/document/d/${id}/edit?usp=drive_open` : `/file/d/${id}/view?usp=drive_open`
|
|
110
|
+
return new Response(null, { status: 307, headers: { location, 'content-type': 'application/binary' } })
|
|
111
|
+
}
|
|
112
|
+
if (/^\/(drive\/folders|file\/d)\//.test(url.pathname)) return html('<!DOCTYPE html><html><head><title>Drive</title></head><body><script>app()</script></body></html>')
|
|
113
|
+
return notFound()
|
|
114
|
+
})
|
|
115
|
+
|
|
116
|
+
origins.usercontent = serve('usercontent', (url, request) => {
|
|
117
|
+
const item = itemOf(url.searchParams.get('id') ?? '')
|
|
118
|
+
if (url.pathname !== '/download' || item?.kind !== 'file' || !keyed(item, url)) return notFound()
|
|
119
|
+
if (item.private === true) return signIn()
|
|
120
|
+
if (item.quota === true) {
|
|
121
|
+
return html('<html><head><title>Google Drive - Quota exceeded</title></head><body><p>Sorry, you can\'t view or download this file at this time.</p>'
|
|
122
|
+
+ '<p>Too many users have viewed or downloaded this file recently. Please try accessing the file again later.</p></body></html>')
|
|
123
|
+
}
|
|
124
|
+
if (item.warn === true && url.searchParams.get('uuid') !== WARN_UUID) {
|
|
125
|
+
return html(`<!DOCTYPE html><html><head><title>Google Drive - Virus scan warning</title></head><body><div class="uc-main"><div id="uc-text">`
|
|
126
|
+
+ `<p class="uc-warning-caption">Google Drive can't scan this file for viruses.</p><p class="uc-warning-subcaption"><span class="uc-name-size"><a href="/open?id=${item.id}">${escapeHtml(item.name)}</a> (64K)</span> is too large for Google to scan for viruses. Would you still like to download this file?</p>`
|
|
127
|
+
+ `<form id="download-form" action="${origins.usercontent}/download" method="get"><input type="submit" id="uc-download-link" class="goog-inline-block jfk-button jfk-button-action" value="Download anyway"/>`
|
|
128
|
+
+ `<input type="hidden" name="id" value="${item.id}"><input type="hidden" name="export" value="download"><input type="hidden" name="confirm" value="t"><input type="hidden" name="uuid" value="${WARN_UUID}"></form></div></div></body></html>`)
|
|
129
|
+
}
|
|
130
|
+
const bytes = bytesOf(item)
|
|
131
|
+
const ascii = /^[\x20-\x7e]*$/.test(item.name)
|
|
132
|
+
const headers: Record<string, string> = {
|
|
133
|
+
'content-type': 'application/octet-stream',
|
|
134
|
+
'content-disposition': ascii ? `attachment; filename="${item.name}"` : `attachment; filename="${item.name.replace(/[^\x20-\x7e]/g, '_')}"; filename*=UTF-8''${encodeURIComponent(item.name)}`,
|
|
135
|
+
}
|
|
136
|
+
const range = /^bytes=(\d+)-(\d*)$/.exec(request.headers.get('range') ?? '')
|
|
137
|
+
if (range != null) {
|
|
138
|
+
const start = Number(range[1])
|
|
139
|
+
const end = Math.min(range[2] !== '' ? Number(range[2]) : bytes.length - 1, bytes.length - 1)
|
|
140
|
+
return new Response(bytes.slice(start, end + 1), { status: 206, headers: { ...headers, 'content-range': `bytes ${start}-${end}/${bytes.length}` } })
|
|
141
|
+
}
|
|
142
|
+
return new Response(bytes, { status: 200, headers })
|
|
143
|
+
})
|
|
144
|
+
|
|
145
|
+
origins.docs = serve('docs', url => {
|
|
146
|
+
const exported = /^\/exported\/([^/]+)\/([a-z]+)$/.exec(url.pathname)
|
|
147
|
+
if (exported != null) {
|
|
148
|
+
const item = itemOf(exported[1]!)
|
|
149
|
+
const format = item?.exports?.[exported[2]!]
|
|
150
|
+
if (item == null || format == null) return notFound()
|
|
151
|
+
return new Response(format.body, {
|
|
152
|
+
status: 200,
|
|
153
|
+
headers: { 'content-type': format.mime, 'content-disposition': `attachment; filename="${item.name}.${exported[2]}"; filename*=UTF-8''${encodeURIComponent(item.name)}.${exported[2]}` },
|
|
154
|
+
})
|
|
155
|
+
}
|
|
156
|
+
const doc = /^\/document\/d\/([^/]+)\/(export|edit)$/.exec(url.pathname)
|
|
157
|
+
const item = doc != null ? itemOf(doc[1]!) : undefined
|
|
158
|
+
if (doc == null || item?.kind !== 'document') return notFound()
|
|
159
|
+
if (item.private === true) return signIn()
|
|
160
|
+
if (doc[2] === 'edit') return html('<!DOCTYPE html><html><head><title>Spec - Google Docs</title></head><body><script>docs()</script></body></html>')
|
|
161
|
+
// Docs answers an export with a redirect to a googleusercontent host that serves the bytes.
|
|
162
|
+
return new Response(null, { status: 307, headers: { location: `/exported/${item.id}/${url.searchParams.get('format') ?? 'docx'}` } })
|
|
163
|
+
})
|
|
164
|
+
|
|
165
|
+
origins.accounts = serve('accounts', () => html('<!DOCTYPE html><html><head><title>Sign in - Google Accounts</title></head><body>Sign in to continue to Google Drive</body></html>'))
|
|
166
|
+
|
|
167
|
+
const apiError = (status: number, reason: string, message: string, details?: string): Response =>
|
|
168
|
+
new Response(JSON.stringify({
|
|
169
|
+
error: { code: status, message, errors: [{ message, domain: 'global', reason }], ...(details != null ? { status: 'INVALID_ARGUMENT', details: [{ reason: details }] } : {}) },
|
|
170
|
+
}), { status, headers: { 'content-type': 'application/json; charset=UTF-8' } })
|
|
171
|
+
|
|
172
|
+
const apiFile = (item: StandInItem): Record<string, unknown> => ({
|
|
173
|
+
id: item.id, name: item.name, mimeType: GOOGLE_TYPES[item.kind] ?? item.mime ?? 'application/octet-stream',
|
|
174
|
+
// Drive shows its warning form only past its scan limit — the API declares such a size.
|
|
175
|
+
...(item.kind === 'file' ? { size: String(item.warn === true ? WARN_DECLARED_SIZE : bytesOf(item).length) } : {}),
|
|
176
|
+
...(item.resourceKey != null ? { resourceKey: item.resourceKey } : {}),
|
|
177
|
+
})
|
|
178
|
+
|
|
179
|
+
origins.api = serve('api', (url, request) => {
|
|
180
|
+
const key = request.headers.get('x-goog-api-key')
|
|
181
|
+
if (key !== API_KEY) return apiError(400, 'badRequest', 'API key not valid. Please pass a valid API key.', 'API_KEY_INVALID')
|
|
182
|
+
if (url.pathname === '/drive/v3/files') {
|
|
183
|
+
const parent = /^'([^']+)' in parents and trashed=false$/.exec(url.searchParams.get('q') ?? '')?.[1]
|
|
184
|
+
const folder = parent != null ? itemOf(parent) : undefined
|
|
185
|
+
if (folder?.kind !== 'folder' || folder.private === true) return apiError(404, 'notFound', `File not found: ${parent}.`)
|
|
186
|
+
const children = childrenOf(folder.id).filter(item => item.private !== true)
|
|
187
|
+
const start = Number(url.searchParams.get('pageToken') ?? '0')
|
|
188
|
+
const page = children.slice(start, start + 2)
|
|
189
|
+
return Response.json({
|
|
190
|
+
files: page.map(apiFile),
|
|
191
|
+
...(start + 2 < children.length ? { nextPageToken: String(start + 2) } : {}),
|
|
192
|
+
})
|
|
193
|
+
}
|
|
194
|
+
const file = /^\/drive\/v3\/files\/([^/]+)$/.exec(url.pathname)
|
|
195
|
+
const item = file != null ? itemOf(decodeURIComponent(file[1]!)) : undefined
|
|
196
|
+
if (item == null || item.private === true) return apiError(404, 'notFound', 'File not found.')
|
|
197
|
+
return Response.json(apiFile(item))
|
|
198
|
+
})
|
|
199
|
+
|
|
200
|
+
const short = serve('short', () => new Response(null, { status: 302, headers: { location: `${origins.drive}/drive/folders/${ROOT}?usp=sharing` } }))
|
|
201
|
+
|
|
202
|
+
return {
|
|
203
|
+
origins, short: `${short}/s/project-files`, hits,
|
|
204
|
+
stop: () => servers.forEach(server => void server.stop(true)),
|
|
205
|
+
}
|
|
206
|
+
}
|
|
207
|
+
|
|
208
|
+
/** The source seam over plain `fetch` (redirects followed, `response.url` set) — what `expand` needs, without a reader. */
|
|
209
|
+
export const plainContext = (): ReaderSourceContext => ({
|
|
210
|
+
fetch: (url, init = {}) => fetch(url, { ...(init.headers != null ? { headers: init.headers } : {}), ...(init.signal != null ? { signal: init.signal } : {}) }),
|
|
211
|
+
limits: { ...DEFAULT_COLLECT_LIMITS },
|
|
212
|
+
})
|
|
@@ -0,0 +1,88 @@
|
|
|
1
|
+
/** One item the Drive stand-in serves. */
|
|
2
|
+
export interface StandInItem {
|
|
3
|
+
id: string
|
|
4
|
+
/** The folder it is listed in. */
|
|
5
|
+
parent?: string
|
|
6
|
+
kind: 'folder' | 'file' | 'document' | 'form'
|
|
7
|
+
name: string
|
|
8
|
+
mime?: string
|
|
9
|
+
body?: string | Uint8Array
|
|
10
|
+
/** Doc exports by format. */
|
|
11
|
+
exports?: Record<string, { mime: string, body: string }>
|
|
12
|
+
/** Shared privately: Google answers with its sign-in page. */
|
|
13
|
+
private?: boolean
|
|
14
|
+
/** Download quota exhausted. */
|
|
15
|
+
quota?: boolean
|
|
16
|
+
/** Served only after Drive's "can't scan for viruses" form (its `uuid`). */
|
|
17
|
+
warn?: boolean
|
|
18
|
+
/** The folder view never answers. */
|
|
19
|
+
stall?: boolean
|
|
20
|
+
/** A pre-2021 link: the item is reachable only with its resource key. */
|
|
21
|
+
resourceKey?: string
|
|
22
|
+
}
|
|
23
|
+
|
|
24
|
+
/** Bytes that are not text. */
|
|
25
|
+
const pattern = (size: number): Uint8Array => Uint8Array.from({ length: size }, (_, index) => (index * 31 + 7) % 251)
|
|
26
|
+
|
|
27
|
+
export const PNG = Uint8Array.from([0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a, 0, 0, 0, 13, 0x49, 0x48, 0x44, 0x52, 0, 0, 0, 1, 0, 0, 0, 1, 8, 6, 0, 0, 0])
|
|
28
|
+
|
|
29
|
+
export const ROOT = 'rootFolder0001'
|
|
30
|
+
export const ASSETS = 'subAssets0001'
|
|
31
|
+
export const DEEP = 'subDeep000001'
|
|
32
|
+
export const WIDE = 'wideFolder001'
|
|
33
|
+
export const PRIVATE_FOLDER = 'privateFold01'
|
|
34
|
+
export const STALL_FOLDER = 'stallFolder01'
|
|
35
|
+
export const KEYED_FOLDER = 'keyedFolder01'
|
|
36
|
+
export const KEYED_RESOURCE = '0-keyedResKey'
|
|
37
|
+
export const LOGO = 'fileLogo00001'
|
|
38
|
+
export const BIG = 'fileBig000001'
|
|
39
|
+
export const SPEC = 'docSpec000001'
|
|
40
|
+
export const SECRET = 'fileSecret001'
|
|
41
|
+
export const QUOTA = 'fileQuota0001'
|
|
42
|
+
export const ACCENTED = 'fileAccent001'
|
|
43
|
+
export const MISSING = 'missingItem01'
|
|
44
|
+
|
|
45
|
+
export const BIG_BODY = pattern(64 * 1024)
|
|
46
|
+
|
|
47
|
+
export const ITEMS: StandInItem[] = [
|
|
48
|
+
{ id: ROOT, kind: 'folder', name: 'Project' },
|
|
49
|
+
{ id: 'fileNotes0001', parent: ROOT, kind: 'file', name: 'notes.md', mime: 'text/markdown', body: '# Notes\n\nPlain words.\n' },
|
|
50
|
+
{ id: LOGO, parent: ROOT, kind: 'file', name: 'logo.svg', mime: 'image/svg+xml', body: '<svg xmlns="http://www.w3.org/2000/svg" width="1" height="1"/>' },
|
|
51
|
+
{ id: 'fileIndex0001', parent: ROOT, kind: 'file', name: 'index.html', mime: 'text/html', body: '<!doctype html><html><body><script>alert(1)</script>Kept as served</body></html>' },
|
|
52
|
+
{ id: BIG, parent: ROOT, kind: 'file', name: 'big.bin', mime: 'application/octet-stream', body: BIG_BODY, warn: true },
|
|
53
|
+
{
|
|
54
|
+
id: SPEC, parent: ROOT, kind: 'document', name: 'Spec',
|
|
55
|
+
exports: {
|
|
56
|
+
docx: { mime: 'application/vnd.openxmlformats-officedocument.wordprocessingml.document', body: 'PK-docx-of-Spec' },
|
|
57
|
+
md: { mime: 'text/markdown', body: '# Spec\n\nExported as Markdown.\n' },
|
|
58
|
+
},
|
|
59
|
+
},
|
|
60
|
+
{ id: 'formPoll00001', parent: ROOT, kind: 'form', name: 'Poll' },
|
|
61
|
+
{ id: SECRET, parent: ROOT, kind: 'file', name: 'secret.txt', mime: 'text/plain', body: 'secret', private: true },
|
|
62
|
+
{ id: ASSETS, parent: ROOT, kind: 'folder', name: 'assets' },
|
|
63
|
+
{ id: 'filePhoto0001', parent: ASSETS, kind: 'file', name: 'photo.png', mime: 'image/png', body: PNG },
|
|
64
|
+
{ id: DEEP, parent: ASSETS, kind: 'folder', name: 'deep' },
|
|
65
|
+
{ id: 'fileDeep00001', parent: DEEP, kind: 'file', name: 'deep.txt', mime: 'text/plain', body: 'deep\n' },
|
|
66
|
+
|
|
67
|
+
{ id: WIDE, kind: 'folder', name: 'Wide' },
|
|
68
|
+
...Array.from({ length: 12 }, (_, index): StandInItem => ({
|
|
69
|
+
id: `fileWide${String(index).padStart(5, '0')}`, parent: WIDE, kind: 'file', name: `wide-${String(index).padStart(2, '0')}.txt`,
|
|
70
|
+
mime: 'text/plain', body: `wide ${index}\n`,
|
|
71
|
+
})),
|
|
72
|
+
|
|
73
|
+
{ id: KEYED_FOLDER, kind: 'folder', name: 'Keyed', resourceKey: KEYED_RESOURCE },
|
|
74
|
+
{ id: 'fileKeyed0001', parent: KEYED_FOLDER, kind: 'file', name: 'keyed.txt', mime: 'text/plain', body: 'keyed\n' },
|
|
75
|
+
|
|
76
|
+
{ id: PRIVATE_FOLDER, kind: 'folder', name: 'Hidden', private: true },
|
|
77
|
+
{ id: STALL_FOLDER, kind: 'folder', name: 'Slow', stall: true },
|
|
78
|
+
{ id: QUOTA, kind: 'file', name: 'popular.zip', mime: 'application/zip', body: 'zip', quota: true },
|
|
79
|
+
{ id: ACCENTED, kind: 'file', name: 'résumé 2026.txt', mime: 'text/plain', body: 'cv\n' },
|
|
80
|
+
]
|
|
81
|
+
|
|
82
|
+
/** The bytes of an item as served. */
|
|
83
|
+
export const bytesOf = (item: StandInItem): Uint8Array =>
|
|
84
|
+
typeof item.body === 'string' ? new TextEncoder().encode(item.body) : item.body ?? new Uint8Array()
|
|
85
|
+
|
|
86
|
+
export const itemOf = (id: string): StandInItem | undefined => ITEMS.find(item => item.id === id)
|
|
87
|
+
|
|
88
|
+
export const childrenOf = (id: string): StandInItem[] => ITEMS.filter(item => item.parent === id)
|
|
@@ -0,0 +1,53 @@
|
|
|
1
|
+
import { afterAll, describe, expect, test } from 'bun:test'
|
|
2
|
+
import { createHash } from 'node:crypto'
|
|
3
|
+
import { mkdtemp, readdir, readFile, rm } from 'node:fs/promises'
|
|
4
|
+
import { tmpdir } from 'node:os'
|
|
5
|
+
import { join } from 'node:path'
|
|
6
|
+
import { CollectKind, CollectStatus, createDirectoryCollectSink, createLlmReader } from '@owlmeans/llm-reader'
|
|
7
|
+
import { isSkip } from '@owlmeans/test'
|
|
8
|
+
import { createGoogleDriveSource, GOOGLE_DRIVE_SOURCE } from '../src/index.js'
|
|
9
|
+
import { gates } from './context.js'
|
|
10
|
+
|
|
11
|
+
const gate = gates.live
|
|
12
|
+
const it = isSkip(gate) ? test.skip : test
|
|
13
|
+
const skipped = isSkip(gate) ? ` (skipped: ${gate.reason})` : ''
|
|
14
|
+
|
|
15
|
+
const FOLDER = 'https://drive.google.com/drive/folders/1knWJ30Rs2f6R8QEdch2I9WMQ8xU7k7_G?usp=share_link'
|
|
16
|
+
const FILE = 'https://drive.google.com/file/d/14ensN8ZJqqoWdCJjtOB31bMOz39wy1N3/view?usp=drive_web'
|
|
17
|
+
const SHA256: Record<string, string> = {
|
|
18
|
+
'crmore-benefits.md': '3e5a0af1e1c9b30883135ae4df70d63039e64687666c1227f7c5585cec0dfe2c',
|
|
19
|
+
'crmore-logo.svg': 'f96de2f591701b9939799d3f3f2867737267b60fbd0088bc60bd49e70c34f72d',
|
|
20
|
+
'index.html': '64cde9b44deeb37d28f7cbd0a1badee5c34af2c5d53617b45c4753fd489cb158',
|
|
21
|
+
}
|
|
22
|
+
|
|
23
|
+
const scratch = await mkdtemp(join(tmpdir(), 'llm-reader-google-drive-live-'))
|
|
24
|
+
afterAll(async () => {
|
|
25
|
+
await rm(scratch, { recursive: true, force: true })
|
|
26
|
+
})
|
|
27
|
+
|
|
28
|
+
const digests = async (root: string): Promise<Record<string, string>> => {
|
|
29
|
+
const names = (await readdir(root, { recursive: true, withFileTypes: true })).filter(entry => entry.isFile())
|
|
30
|
+
.map(entry => join(entry.parentPath, entry.name).slice(root.length + 1))
|
|
31
|
+
return Object.fromEntries(await Promise.all(names.map(async name =>
|
|
32
|
+
[name, createHash('sha256').update(await readFile(join(root, name))).digest('hex')] as const)))
|
|
33
|
+
}
|
|
34
|
+
|
|
35
|
+
/** The public Drive, keyless, through a real reader with its default fetch — `LLM_READER_GDRIVE_LIVE=1`. */
|
|
36
|
+
describe(`@owlmeans/llm-reader-google-drive — live Google Drive${skipped}`, () => {
|
|
37
|
+
const reader = createLlmReader({ sources: [createGoogleDriveSource()] })
|
|
38
|
+
|
|
39
|
+
it('a shared folder is exactly its three files, byte for byte', async () => {
|
|
40
|
+
const root = join(scratch, 'folder')
|
|
41
|
+
const result = await reader.collect(FOLDER, { sink: createDirectoryCollectSink(root) })
|
|
42
|
+
expect(result).toMatchObject({ kind: CollectKind.Source, source: GOOGLE_DRIVE_SOURCE, files: 3 })
|
|
43
|
+
expect(result.items.every(item => item.status === CollectStatus.Written)).toBe(true)
|
|
44
|
+
expect(await digests(root)).toEqual(SHA256)
|
|
45
|
+
}, 120_000)
|
|
46
|
+
|
|
47
|
+
it('a single shared file is that file, named as Drive names it', async () => {
|
|
48
|
+
const root = join(scratch, 'file')
|
|
49
|
+
const result = await reader.collect(FILE, { sink: createDirectoryCollectSink(root) })
|
|
50
|
+
expect(result).toMatchObject({ kind: CollectKind.Source, entry: 'crmore-logo.svg', files: 1 })
|
|
51
|
+
expect(await digests(root)).toEqual({ 'crmore-logo.svg': SHA256['crmore-logo.svg'] })
|
|
52
|
+
}, 120_000)
|
|
53
|
+
})
|
|
@@ -0,0 +1,189 @@
|
|
|
1
|
+
import { afterAll, describe, expect, test } from 'bun:test'
|
|
2
|
+
import { mkdtemp, readFile, readdir, rm } from 'node:fs/promises'
|
|
3
|
+
import { tmpdir } from 'node:os'
|
|
4
|
+
import { join } from 'node:path'
|
|
5
|
+
import { CollectKind, CollectStatus, createDirectoryCollectSink, createLlmReader } from '@owlmeans/llm-reader'
|
|
6
|
+
import type { CollectSink, LlmReader } from '@owlmeans/llm-reader'
|
|
7
|
+
import {
|
|
8
|
+
createGoogleDriveSource, createGoogleDriveUrlHelper, GOOGLE_DRIVE_SOURCE, GoogleDriveError, GoogleDriveFailure, GoogleDriveKind,
|
|
9
|
+
} from '../src/index.js'
|
|
10
|
+
import type { GoogleDriveSourceOptions } from '../src/index.js'
|
|
11
|
+
import { API_KEY, plainContext, startDriveStandIn, WARN_DECLARED_SIZE, WARN_UUID } from './context.js'
|
|
12
|
+
import {
|
|
13
|
+
ACCENTED, BIG, BIG_BODY, KEYED_FOLDER, KEYED_RESOURCE, LOGO, MISSING, PNG, PRIVATE_FOLDER, QUOTA, ROOT, SECRET, SPEC, STALL_FOLDER, WIDE,
|
|
14
|
+
} from './fixtures.js'
|
|
15
|
+
|
|
16
|
+
const drive = startDriveStandIn()
|
|
17
|
+
const scratch = await mkdtemp(join(tmpdir(), 'llm-reader-google-drive-'))
|
|
18
|
+
afterAll(async () => {
|
|
19
|
+
drive.stop()
|
|
20
|
+
await rm(scratch, { recursive: true, force: true })
|
|
21
|
+
})
|
|
22
|
+
|
|
23
|
+
let dirs = 0
|
|
24
|
+
const target = (): { root: string, sink: CollectSink } => {
|
|
25
|
+
const root = join(scratch, `run-${++dirs}`)
|
|
26
|
+
return { root, sink: createDirectoryCollectSink(root) }
|
|
27
|
+
}
|
|
28
|
+
const files = async (root: string): Promise<string[]> =>
|
|
29
|
+
(await readdir(root, { recursive: true, withFileTypes: true })).filter(entry => entry.isFile())
|
|
30
|
+
.map(entry => join(entry.parentPath, entry.name).slice(root.length + 1)).sort()
|
|
31
|
+
|
|
32
|
+
/** A real reader with the source pointed at the stand-in; the guard allows only the loopback stand-in. */
|
|
33
|
+
const readerWith = (options: GoogleDriveSourceOptions = {}): LlmReader =>
|
|
34
|
+
createLlmReader({ sources: [createGoogleDriveSource({ origins: drive.origins, ...options })], allowHosts: ['127.0.0.1'], timeout: 1500, s3: false })
|
|
35
|
+
|
|
36
|
+
const reader = readerWith()
|
|
37
|
+
const folderLink = (id: string, query = '?usp=share_link'): string => `${drive.origins.drive}/drive/folders/${id}${query}`
|
|
38
|
+
const fileLink = (id: string): string => `${drive.origins.drive}/file/d/${id}/view?usp=drive_web`
|
|
39
|
+
|
|
40
|
+
const ROOT_FILES = ['Spec.docx', 'assets/deep/deep.txt', 'assets/photo.png', 'big.bin', 'index.html', 'logo.svg', 'notes.md']
|
|
41
|
+
|
|
42
|
+
describe('@owlmeans/llm-reader-google-drive', () => {
|
|
43
|
+
test('Drive and Docs links are recognised from the URL alone; URLs are built for downloads, exports and listings', () => {
|
|
44
|
+
const source = createGoogleDriveSource()
|
|
45
|
+
const id = '1knWJ30Rs2f6R8QEdch2I9WMQ8xU7k7_G'
|
|
46
|
+
for (const url of [
|
|
47
|
+
`https://drive.google.com/drive/folders/${id}?usp=share_link`, `https://drive.google.com/drive/u/1/folders/${id}`,
|
|
48
|
+
`https://drive.google.com/drive/folders/${id}?resourcekey=0-abc`, `https://drive.google.com/file/d/${id}/view?usp=drive_web`,
|
|
49
|
+
`https://drive.google.com/open?id=${id}`, `https://drive.google.com/uc?id=${id}&export=download`, `http://drive.google.com/file/d/${id}`,
|
|
50
|
+
`https://drive.usercontent.google.com/download?id=${id}&export=download`, `https://docs.google.com/document/d/${id}/edit`,
|
|
51
|
+
`https://docs.google.com/spreadsheets/d/${id}/edit#gid=0`, `https://docs.google.com/presentation/d/${id}/edit`,
|
|
52
|
+
`https://docs.google.com/drawings/d/${id}/edit`, `https://docs.google.com/document/u/0/d/${id}/edit`,
|
|
53
|
+
]) expect([url, source.matches(url)]).toEqual([url, true])
|
|
54
|
+
for (const url of [
|
|
55
|
+
'https://drive.google.com/', 'https://drive.google.com/drive/my-drive', `https://docs.google.com/forms/d/${id}/viewform`,
|
|
56
|
+
`https://docs.google.com/document/d/e/2PACX-1vQx/pub`, 'https://drive.google.com/drive/folders/short', `https://example.com/drive/folders/${id}`,
|
|
57
|
+
`ftp://drive.google.com/file/d/${id}`, 'not a url',
|
|
58
|
+
]) expect([url, source.matches(url)]).toEqual([url, false])
|
|
59
|
+
|
|
60
|
+
const urls = createGoogleDriveUrlHelper({ exports: { document: 'md', spreadsheet: 'csv' } })
|
|
61
|
+
expect(urls.parse(`https://drive.google.com/drive/folders/${id}?resourcekey=0-abc`)).toEqual({ kind: GoogleDriveKind.Folder, id, resourceKey: '0-abc' })
|
|
62
|
+
expect(urls.parse(`https://drive.google.com/open?id=${id}`)).toEqual({ kind: GoogleDriveKind.Unresolved, id })
|
|
63
|
+
expect(urls.parse(`https://docs.google.com/spreadsheets/d/${id}/edit`)).toEqual({ kind: GoogleDriveKind.Spreadsheet, id })
|
|
64
|
+
expect(urls.downloadUrl(id)).toBe(`https://drive.usercontent.google.com/download?id=${id}&export=download&confirm=t`)
|
|
65
|
+
expect(urls.exportOf(GoogleDriveKind.Document, id)).toEqual({ url: `https://docs.google.com/document/d/${id}/export?format=md`, extension: 'md' })
|
|
66
|
+
expect(urls.exportOf(GoogleDriveKind.Presentation, id).url).toBe(`https://docs.google.com/presentation/d/${id}/export/pptx`)
|
|
67
|
+
expect(urls.folderViewUrl(id, '0-abc')).toBe(`https://drive.google.com/embeddedfolderview?id=${id}&resourcekey=0-abc`)
|
|
68
|
+
})
|
|
69
|
+
|
|
70
|
+
test('a public folder is collected through a real reader: sub-folders, an exported Doc, the warning form followed, private and form items left out', async () => {
|
|
71
|
+
const { root, sink } = target()
|
|
72
|
+
const before = drive.hits.length
|
|
73
|
+
const result = await reader.collect(folderLink(ROOT), { sink, prefix: 'links/01-drive' })
|
|
74
|
+
expect(result).toMatchObject({ kind: CollectKind.Source, source: GOOGLE_DRIVE_SOURCE, files: ROOT_FILES.length })
|
|
75
|
+
expect(result.items.every(item => item.status === CollectStatus.Written)).toBe(true)
|
|
76
|
+
expect(await files(root)).toEqual(ROOT_FILES.map(path => `links/01-drive/${path}`))
|
|
77
|
+
const at = (path: string): Promise<Buffer> => readFile(join(root, 'links/01-drive', path))
|
|
78
|
+
expect(Buffer.compare(await at('big.bin'), BIG_BODY)).toBe(0)
|
|
79
|
+
expect(new Uint8Array(await at('assets/photo.png'))).toEqual(PNG)
|
|
80
|
+
expect((await at('Spec.docx')).toString()).toBe('PK-docx-of-Spec')
|
|
81
|
+
// An HTML file is a file like any other: kept as served, never bundled.
|
|
82
|
+
expect((await at('index.html')).toString()).toContain('<script>alert(1)</script>')
|
|
83
|
+
const items = Object.fromEntries(result.items.map(item => [item.path, item]))
|
|
84
|
+
expect(items['links/01-drive/notes.md']?.mime).toBe('text/markdown')
|
|
85
|
+
expect(items['links/01-drive/big.bin']?.url).toContain(`uuid=${WARN_UUID}`)
|
|
86
|
+
|
|
87
|
+
const hits = drive.hits.slice(before)
|
|
88
|
+
// Probes are ranged; the collector's own downloads are not.
|
|
89
|
+
const probes = hits.filter(hit => hit.role === 'usercontent' && hit.headers.range === 'bytes=0-0')
|
|
90
|
+
expect(probes.length).toBeGreaterThanOrEqual(6)
|
|
91
|
+
expect(hits.filter(hit => hit.role === 'drive' && hit.url.startsWith('/embeddedfolderview')).length).toBe(3)
|
|
92
|
+
expect(hits.some(hit => hit.url.includes(SECRET) && hit.headers.range == null)).toBe(false)
|
|
93
|
+
}, 30_000)
|
|
94
|
+
|
|
95
|
+
test('depth, maxEntries and the collect\'s maxFiles bound the walk; the cut is reported', async () => {
|
|
96
|
+
const shallow = await readerWith({ depth: 0 }).collect(folderLink(ROOT), target())
|
|
97
|
+
expect(shallow.items.map(item => item.path).sort()).toEqual(['Spec.docx', 'big.bin', 'index.html', 'logo.svg', 'notes.md'])
|
|
98
|
+
const one = await readerWith({ depth: 1 }).collect(folderLink(ROOT), target())
|
|
99
|
+
expect(one.items.map(item => item.path)).toContain('assets/photo.png')
|
|
100
|
+
expect(one.items.map(item => item.path)).not.toContain('assets/deep/deep.txt')
|
|
101
|
+
|
|
102
|
+
const limited = await reader.collect(folderLink(WIDE), { ...target(), limits: { maxFiles: 5 } })
|
|
103
|
+
expect(limited.items.filter(item => item.status === CollectStatus.Written)).toHaveLength(5)
|
|
104
|
+
expect(limited.items.filter(item => item.status === CollectStatus.Skipped).map(item => item.reason)).toEqual(['max-files'])
|
|
105
|
+
const capped = await readerWith({ maxEntries: 3 }).collect(folderLink(WIDE), target())
|
|
106
|
+
expect(capped.items.map(item => item.path)).toEqual(['wide-00.txt', 'wide-01.txt', 'wide-02.txt'])
|
|
107
|
+
}, 30_000)
|
|
108
|
+
|
|
109
|
+
test('editor exports follow the options; single links are one entry named as Drive names it', async () => {
|
|
110
|
+
const { root, sink } = target()
|
|
111
|
+
const markdown = await readerWith({ exports: { document: 'md' } }).collect(folderLink(ROOT), { sink, prefix: 'md' })
|
|
112
|
+
expect(markdown.items.map(item => item.path)).toContain('md/Spec.md')
|
|
113
|
+
expect((await readFile(join(root, 'md/Spec.md'), 'utf8'))).toBe('# Spec\n\nExported as Markdown.\n')
|
|
114
|
+
|
|
115
|
+
const single = async (url: string): Promise<{ entry?: string, kind: CollectKind, bytes: number }> => {
|
|
116
|
+
const result = await reader.collect(url, target())
|
|
117
|
+
expect(result.items.every(item => item.status === CollectStatus.Written)).toBe(true)
|
|
118
|
+
return { ...(result.entry != null ? { entry: result.entry } : {}), kind: result.kind, bytes: result.bytes }
|
|
119
|
+
}
|
|
120
|
+
expect(await single(fileLink(LOGO))).toMatchObject({ entry: 'logo.svg', kind: CollectKind.Source })
|
|
121
|
+
expect(await single(`${drive.origins.drive}/uc?id=${LOGO}&export=download`)).toMatchObject({ entry: 'logo.svg' })
|
|
122
|
+
expect(await single(`${drive.origins.drive}/open?id=${LOGO}`)).toMatchObject({ entry: 'logo.svg' })
|
|
123
|
+
expect(await single(fileLink(ACCENTED))).toMatchObject({ entry: 'résumé 2026.txt' })
|
|
124
|
+
expect(await single(fileLink(BIG))).toMatchObject({ entry: 'big.bin', bytes: BIG_BODY.length })
|
|
125
|
+
expect(await single(`${drive.origins.docs}/document/d/${SPEC}/edit?usp=sharing`)).toMatchObject({ entry: 'Spec.docx' })
|
|
126
|
+
const opened = await reader.collect(`${drive.origins.drive}/open?id=${ROOT}`, target())
|
|
127
|
+
expect(opened.files).toBe(ROOT_FILES.length)
|
|
128
|
+
// A link that lands on Drive through a redirect (a shortener) is taken by where it landed.
|
|
129
|
+
const shortened = await reader.collect(drive.short, target())
|
|
130
|
+
expect(shortened).toMatchObject({ kind: CollectKind.Source, source: GOOGLE_DRIVE_SOURCE, files: ROOT_FILES.length })
|
|
131
|
+
// A pre-2021 link carries its resource key to the folder view.
|
|
132
|
+
const keyed = await reader.collect(folderLink(KEYED_FOLDER, `?resourcekey=${KEYED_RESOURCE}`), target())
|
|
133
|
+
expect(keyed.items.map(item => [item.path, item.status])).toEqual([['keyed.txt', CollectStatus.Written]])
|
|
134
|
+
}, 30_000)
|
|
135
|
+
|
|
136
|
+
test('refusals are clear errors: private → not-public, quota, 404 → not-found; a stalled answer times out', async () => {
|
|
137
|
+
const source = createGoogleDriveSource({ origins: drive.origins })
|
|
138
|
+
const reason = async (url: string): Promise<GoogleDriveFailure | string> => {
|
|
139
|
+
const error = await source.expand(url, plainContext()).then(() => 'expanded', (e: unknown) => e)
|
|
140
|
+
return error instanceof GoogleDriveError ? error.reason : String(error)
|
|
141
|
+
}
|
|
142
|
+
expect(await reason(folderLink(PRIVATE_FOLDER))).toBe(GoogleDriveFailure.NotPublic)
|
|
143
|
+
expect(await reason(fileLink(SECRET))).toBe(GoogleDriveFailure.NotPublic)
|
|
144
|
+
expect(await reason(fileLink(QUOTA))).toBe(GoogleDriveFailure.Quota)
|
|
145
|
+
expect(await reason(folderLink(MISSING))).toBe(GoogleDriveFailure.NotFound)
|
|
146
|
+
expect(await reason(folderLink(KEYED_FOLDER))).toBe(GoogleDriveFailure.NotFound)
|
|
147
|
+
expect(await reason('https://example.com/file')).toBe(GoogleDriveFailure.NotALink)
|
|
148
|
+
|
|
149
|
+
const refused = await reader.collect(folderLink(PRIVATE_FOLDER), target())
|
|
150
|
+
expect(refused).toMatchObject({ kind: CollectKind.Source, files: 0 })
|
|
151
|
+
expect(refused.items).toEqual([expect.objectContaining({ status: CollectStatus.Failed, reason: 'llm-reader:fetch:google-drive:not-public' })])
|
|
152
|
+
const started = Date.now()
|
|
153
|
+
const stalled = await reader.collect(folderLink(STALL_FOLDER), target())
|
|
154
|
+
expect(stalled.items[0]).toMatchObject({ status: CollectStatus.Failed, reason: 'llm-reader:fetch:timeout' })
|
|
155
|
+
expect(Date.now() - started).toBeLessThan(5_000)
|
|
156
|
+
}, 30_000)
|
|
157
|
+
|
|
158
|
+
test('with an API key: Drive API pages with sizes and media types, the key only as a header, a refused key is api-key', async () => {
|
|
159
|
+
const before = drive.hits.length
|
|
160
|
+
const source = createGoogleDriveSource({ origins: drive.origins, apiKey: API_KEY })
|
|
161
|
+
const entries = await source.expand(folderLink(ROOT), plainContext())
|
|
162
|
+
// The file behind the warning form declares a size past the default file cap: not probed, admitted by size.
|
|
163
|
+
expect(entries.map(entry => entry.path).sort()).toEqual(ROOT_FILES)
|
|
164
|
+
expect(entries.find(entry => entry.path === 'notes.md')).toMatchObject({ size: 22, mime: 'text/markdown' })
|
|
165
|
+
expect(entries.find(entry => entry.path === 'big.bin')).toMatchObject({ size: WARN_DECLARED_SIZE })
|
|
166
|
+
const apiHits = drive.hits.slice(before).filter(hit => hit.role === 'api')
|
|
167
|
+
expect(apiHits.some(hit => hit.url.includes('pageToken='))).toBe(true)
|
|
168
|
+
expect(apiHits.every(hit => hit.headers['x-goog-api-key'] === API_KEY && !hit.url.includes(API_KEY))).toBe(true)
|
|
169
|
+
expect(drive.hits.slice(before).some(hit => hit.role === 'drive')).toBe(false)
|
|
170
|
+
|
|
171
|
+
const keyed = createLlmReader({ sources: [source], allowHosts: ['127.0.0.1'], timeout: 1500, s3: false })
|
|
172
|
+
const capped = await keyed.collect(folderLink(ROOT), target())
|
|
173
|
+
expect(capped.items.find(item => item.path === 'big.bin')).toMatchObject({ status: CollectStatus.Skipped, reason: 'max-file-bytes' })
|
|
174
|
+
expect(capped.items.filter(item => item.status === CollectStatus.Written)).toHaveLength(ROOT_FILES.length - 1)
|
|
175
|
+
// Past the scan limit and under a raised cap, the file is probed and its warning form followed.
|
|
176
|
+
const { root, sink } = target()
|
|
177
|
+
const raised = await keyed.collect(folderLink(ROOT), { sink, limits: { maxFileBytes: 200 * 1024 * 1024, maxTotalBytes: 400 * 1024 * 1024 } })
|
|
178
|
+
expect(raised.items.find(item => item.path === 'big.bin')).toMatchObject({ status: CollectStatus.Written, bytes: BIG_BODY.length })
|
|
179
|
+
expect(Buffer.compare(await readFile(join(root, 'big.bin')), BIG_BODY)).toBe(0)
|
|
180
|
+
expect((await keyed.collect(`${drive.origins.drive}/open?id=${SPEC}`, target())).entry).toBe('Spec.docx')
|
|
181
|
+
|
|
182
|
+
const wrong = createGoogleDriveSource({ origins: drive.origins, apiKey: 'wrong-key' })
|
|
183
|
+
const error = await wrong.expand(folderLink(ROOT), plainContext()).catch((e: unknown) => e)
|
|
184
|
+
expect(error).toBeInstanceOf(GoogleDriveError)
|
|
185
|
+
expect((error as GoogleDriveError).reason).toBe(GoogleDriveFailure.ApiKey)
|
|
186
|
+
expect(((await source.expand(folderLink(PRIVATE_FOLDER), plainContext()).catch((e: unknown) => e)) as GoogleDriveError).reason)
|
|
187
|
+
.toBe(GoogleDriveFailure.NotFound)
|
|
188
|
+
}, 30_000)
|
|
189
|
+
})
|
package/tsconfig.json
ADDED
|
@@ -0,0 +1,16 @@
|
|
|
1
|
+
{
|
|
2
|
+
"extends": [
|
|
3
|
+
"@owlmeans/dep-config/tsconfig.base.json",
|
|
4
|
+
"@owlmeans/dep-config/tsconfig.node.json"
|
|
5
|
+
],
|
|
6
|
+
"compilerOptions": {
|
|
7
|
+
"rootDir": "./src/",
|
|
8
|
+
"outDir": "./build/"
|
|
9
|
+
},
|
|
10
|
+
"exclude": [
|
|
11
|
+
"./dist/**/*",
|
|
12
|
+
"./build/**/*",
|
|
13
|
+
"./tests/**/*",
|
|
14
|
+
"./*.ts"
|
|
15
|
+
]
|
|
16
|
+
}
|