@owlmeans/llm-reader-google-drive 0.0.0-stage → 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +113 -2
- package/agent-meta/manifest.json +16 -0
- package/agent-meta/skills/llm-reader-google-drive/SKILL.md +131 -0
- package/build/consts.d.ts +45 -0
- package/build/consts.d.ts.map +1 -0
- package/build/consts.js +54 -0
- package/build/consts.js.map +1 -0
- package/build/consts.local.d.ts +3 -0
- package/build/consts.local.d.ts.map +1 -0
- package/build/consts.local.js +3 -0
- package/build/consts.local.js.map +1 -0
- package/build/errors.d.ts +14 -0
- package/build/errors.d.ts.map +1 -0
- package/build/errors.js +26 -0
- package/build/errors.js.map +1 -0
- package/build/index.d.ts +6 -0
- package/build/index.d.ts.map +1 -0
- package/build/index.js +5 -0
- package/build/index.js.map +1 -0
- package/build/source/api.d.ts +9 -0
- package/build/source/api.d.ts.map +1 -0
- package/build/source/api.js +80 -0
- package/build/source/api.js.map +1 -0
- package/build/source/consts.local.d.ts +33 -0
- package/build/source/consts.local.d.ts.map +1 -0
- package/build/source/consts.local.js +39 -0
- package/build/source/consts.local.js.map +1 -0
- package/build/source/embedded.d.ts +10 -0
- package/build/source/embedded.d.ts.map +1 -0
- package/build/source/embedded.js +82 -0
- package/build/source/embedded.js.map +1 -0
- package/build/source/http.d.ts +6 -0
- package/build/source/http.d.ts.map +1 -0
- package/build/source/http.js +61 -0
- package/build/source/http.js.map +1 -0
- package/build/source/probe.d.ts +10 -0
- package/build/source/probe.d.ts.map +1 -0
- package/build/source/probe.js +80 -0
- package/build/source/probe.js.map +1 -0
- package/build/source/types.local.d.ts +87 -0
- package/build/source/types.local.d.ts.map +1 -0
- package/build/source/types.local.js +2 -0
- package/build/source/types.local.js.map +1 -0
- package/build/source.d.ts +22 -0
- package/build/source.d.ts.map +1 -0
- package/build/source.js +169 -0
- package/build/source.js.map +1 -0
- package/build/types.d.ts +79 -0
- package/build/types.d.ts.map +1 -0
- package/build/types.js +2 -0
- package/build/types.js.map +1 -0
- package/build/url/consts.local.d.ts +10 -0
- package/build/url/consts.local.d.ts.map +1 -0
- package/build/url/consts.local.js +15 -0
- package/build/url/consts.local.js.map +1 -0
- package/build/url.d.ts +8 -0
- package/build/url.d.ts.map +1 -0
- package/build/url.js +117 -0
- package/build/url.js.map +1 -0
- package/build/utils/names/types.d.ts +10 -0
- package/build/utils/names/types.d.ts.map +1 -0
- package/build/utils/names/types.js +2 -0
- package/build/utils/names/types.js.map +1 -0
- package/build/utils/names.d.ts +4 -0
- package/build/utils/names.d.ts.map +1 -0
- package/build/utils/names.js +31 -0
- package/build/utils/names.js.map +1 -0
- package/package.json +62 -4
- package/src/consts.local.ts +2 -0
- package/src/consts.ts +60 -0
- package/src/errors.ts +30 -0
- package/src/index.ts +7 -0
- package/src/source/api.ts +81 -0
- package/src/source/consts.local.ts +52 -0
- package/src/source/embedded.ts +83 -0
- package/src/source/http.ts +66 -0
- package/src/source/probe.ts +84 -0
- package/src/source/types.local.ts +91 -0
- package/src/source.ts +179 -0
- package/src/types.ts +86 -0
- package/src/url/consts.local.ts +19 -0
- package/src/url.ts +121 -0
- package/src/utils/names/types.ts +9 -0
- package/src/utils/names.ts +36 -0
- package/tests/context.ts +212 -0
- package/tests/fixtures.ts +88 -0
- package/tests/live.spec.ts +53 -0
- package/tests/source.spec.ts +189 -0
- package/tests/tsconfig.json +12 -0
- package/tsconfig.json +16 -0
package/src/consts.ts
ADDED
|
@@ -0,0 +1,60 @@
|
|
|
1
|
+
import type { GoogleDriveOrigins, GoogleExportFormats } from './types.js'
|
|
2
|
+
|
|
3
|
+
/** The name the source expands links under (`CollectResult.source`). */
|
|
4
|
+
export const GOOGLE_DRIVE_SOURCE = 'google-drive'
|
|
5
|
+
|
|
6
|
+
/** Where Google answers; tests point these at a stand-in. */
|
|
7
|
+
export const DEFAULT_GOOGLE_DRIVE_ORIGINS: Readonly<Required<GoogleDriveOrigins>> = Object.freeze({
|
|
8
|
+
drive: 'https://drive.google.com',
|
|
9
|
+
usercontent: 'https://drive.usercontent.google.com',
|
|
10
|
+
docs: 'https://docs.google.com',
|
|
11
|
+
accounts: 'https://accounts.google.com',
|
|
12
|
+
api: 'https://www.googleapis.com',
|
|
13
|
+
})
|
|
14
|
+
|
|
15
|
+
/** The format each Google editor file is exported as unless `exports` names another. */
|
|
16
|
+
export const DEFAULT_GOOGLE_EXPORTS: Readonly<Required<GoogleExportFormats>> = Object.freeze({
|
|
17
|
+
document: 'docx', spreadsheet: 'xlsx', presentation: 'pptx', drawing: 'png',
|
|
18
|
+
})
|
|
19
|
+
|
|
20
|
+
/** Sub-folder levels listed below the linked folder (`0` = its own files only). */
|
|
21
|
+
export const DEFAULT_GOOGLE_DRIVE_DEPTH = 3
|
|
22
|
+
|
|
23
|
+
/** Entries one link expands to at most — under the collect's own `maxFiles + 1`. */
|
|
24
|
+
export const DEFAULT_GOOGLE_DRIVE_MAX_ENTRIES = 200
|
|
25
|
+
|
|
26
|
+
/** Folders one link lists at most (each is one request). */
|
|
27
|
+
export const DEFAULT_GOOGLE_DRIVE_MAX_FOLDERS = 50
|
|
28
|
+
|
|
29
|
+
/** What a Drive link points at. */
|
|
30
|
+
export enum GoogleDriveKind {
|
|
31
|
+
Folder = 'folder',
|
|
32
|
+
/** A stored file — bytes behind the usercontent download. */
|
|
33
|
+
File = 'file',
|
|
34
|
+
Document = 'document',
|
|
35
|
+
Spreadsheet = 'spreadsheet',
|
|
36
|
+
Presentation = 'presentation',
|
|
37
|
+
Drawing = 'drawing',
|
|
38
|
+
/** An `open?id=` link: Drive itself tells what it is. */
|
|
39
|
+
Unresolved = 'unresolved',
|
|
40
|
+
}
|
|
41
|
+
|
|
42
|
+
/** Why a link or an entry could not be expanded — the `reason` of a `GoogleDriveError`. */
|
|
43
|
+
export enum GoogleDriveFailure {
|
|
44
|
+
/** Shared privately: Google answered with its sign-in page, 401/403 or "you need access". */
|
|
45
|
+
NotPublic = 'not-public',
|
|
46
|
+
/** Download or API quota exhausted ("too many users have viewed or downloaded", 429, rate limits). */
|
|
47
|
+
Quota = 'quota',
|
|
48
|
+
/** Drive answered 404 — no such item, or one an API key may not see. */
|
|
49
|
+
NotFound = 'not-found',
|
|
50
|
+
/** The Drive API refused the key (invalid, or the Drive API not enabled for it). */
|
|
51
|
+
ApiKey = 'api-key',
|
|
52
|
+
/** The URL is not a Drive or Docs link this source reads. */
|
|
53
|
+
NotALink = 'not-a-link',
|
|
54
|
+
/** A Drive answer this source does not know how to read. */
|
|
55
|
+
Unrecognized = 'unrecognized',
|
|
56
|
+
/** A file-streams Drive resolver inside the injected fetch refused the folder page. */
|
|
57
|
+
Resolver = 'resolver',
|
|
58
|
+
/** Any other refusing status (`status:<code>`). */
|
|
59
|
+
Status = 'status',
|
|
60
|
+
}
|
package/src/errors.ts
ADDED
|
@@ -0,0 +1,30 @@
|
|
|
1
|
+
import { ResilientError } from '@owlmeans/error'
|
|
2
|
+
import { ReaderFetchError } from '@owlmeans/llm-reader'
|
|
3
|
+
import { GOOGLE_DRIVE_SOURCE, GoogleDriveFailure } from './consts.js'
|
|
4
|
+
|
|
5
|
+
/**
|
|
6
|
+
* A Drive link or entry that could not be expanded — `llm-reader:fetch:google-drive:<reason>[:<detail>]`.
|
|
7
|
+
* A reader fetch error, so a collect reports it as the link's failed item with this message as the reason.
|
|
8
|
+
*/
|
|
9
|
+
export class GoogleDriveError extends ReaderFetchError {
|
|
10
|
+
public static override typeName = `GoogleDrive${ReaderFetchError.typeName}`
|
|
11
|
+
|
|
12
|
+
/** Why, read back from the message. */
|
|
13
|
+
public reason: GoogleDriveFailure = GoogleDriveFailure.Unrecognized
|
|
14
|
+
|
|
15
|
+
constructor(message: string = GoogleDriveFailure.Unrecognized) {
|
|
16
|
+
super(`${GOOGLE_DRIVE_SOURCE}:${message}`)
|
|
17
|
+
this.type = GoogleDriveError.typeName
|
|
18
|
+
this.finalizeUnmarshal()
|
|
19
|
+
}
|
|
20
|
+
|
|
21
|
+
public override finalizeUnmarshal(): void {
|
|
22
|
+
super.finalizeUnmarshal()
|
|
23
|
+
const reason = new RegExp(`${GOOGLE_DRIVE_SOURCE}:([a-z-]+)`).exec(this.message)?.[1]
|
|
24
|
+
this.reason = Object.values(GoogleDriveFailure).includes(reason as GoogleDriveFailure)
|
|
25
|
+
? reason as GoogleDriveFailure
|
|
26
|
+
: GoogleDriveFailure.Unrecognized
|
|
27
|
+
}
|
|
28
|
+
}
|
|
29
|
+
|
|
30
|
+
ResilientError.registerErrorClass(GoogleDriveError)
|
package/src/index.ts
ADDED
|
@@ -0,0 +1,81 @@
|
|
|
1
|
+
import { logger } from '@owlmeans/log'
|
|
2
|
+
import { GoogleDriveFailure, GoogleDriveKind } from '../consts.js'
|
|
3
|
+
import { DRIVE_ID_PATTERN } from '../consts.local.js'
|
|
4
|
+
import { GoogleDriveError } from '../errors.js'
|
|
5
|
+
import type { GoogleDriveUrlHelper } from '../types.js'
|
|
6
|
+
import { DRIVE_API_MAX_PAGES, DRIVE_LISTING_BYTES, GOOGLE_APPS_KINDS, GOOGLE_APPS_PREFIX, GOOGLE_APPS_SHORTCUT, LOG_SCOPE } from './consts.local.js'
|
|
7
|
+
import type { DriveApiFile, DriveApiList, DriveHttp, DriveItem, DriveLister } from './types.local.js'
|
|
8
|
+
|
|
9
|
+
const log = logger(LOG_SCOPE)
|
|
10
|
+
|
|
11
|
+
/**
|
|
12
|
+
* Folders listed through Drive API v3 with an API key (the `X-Goog-Api-Key` header, never the URL):
|
|
13
|
+
* `files.list` pages of `'<id>' in parents and trashed=false`, with sizes and media types; a
|
|
14
|
+
* shortcut is listed as its target; Google item types other than folders and editor files are left out.
|
|
15
|
+
*/
|
|
16
|
+
export const makeApiLister = (http: DriveHttp, urls: GoogleDriveUrlHelper, apiKey: string): DriveLister => {
|
|
17
|
+
const headersOf = (id: string, resourceKey?: string): Record<string, string> => ({
|
|
18
|
+
'x-goog-api-key': apiKey,
|
|
19
|
+
accept: 'application/json',
|
|
20
|
+
...(resourceKey != null ? { 'x-goog-drive-resource-keys': `${id}/${resourceKey}` } : {}),
|
|
21
|
+
})
|
|
22
|
+
|
|
23
|
+
const json = async <T>(url: string, headers: Record<string, string>): Promise<T> => {
|
|
24
|
+
const response = await http.get(url, { headers, maxBytes: DRIVE_LISTING_BYTES })
|
|
25
|
+
const refused = await http.refusal(response)
|
|
26
|
+
if (refused != null) throw refused
|
|
27
|
+
try {
|
|
28
|
+
return JSON.parse(await response.text()) as T
|
|
29
|
+
} catch {
|
|
30
|
+
throw new GoogleDriveError(`${GoogleDriveFailure.Unrecognized}:api-answer`)
|
|
31
|
+
}
|
|
32
|
+
}
|
|
33
|
+
|
|
34
|
+
const itemOf = (file: DriveApiFile): DriveItem | undefined => {
|
|
35
|
+
const shortcut = file.mimeType === GOOGLE_APPS_SHORTCUT ? file.shortcutDetails : undefined
|
|
36
|
+
const id = shortcut != null ? shortcut.targetId : file.id
|
|
37
|
+
const mime = shortcut != null ? shortcut.targetMimeType : file.mimeType
|
|
38
|
+
const resourceKey = shortcut != null ? shortcut.targetResourceKey : file.resourceKey
|
|
39
|
+
if (id == null || !DRIVE_ID_PATTERN.test(id)) return undefined
|
|
40
|
+
const native = mime != null && mime.startsWith(GOOGLE_APPS_PREFIX)
|
|
41
|
+
const kind = native ? GOOGLE_APPS_KINDS[mime] : GoogleDriveKind.File
|
|
42
|
+
if (kind == null) return undefined
|
|
43
|
+
const size = Number(file.size ?? NaN)
|
|
44
|
+
return {
|
|
45
|
+
id, kind, name: file.name != null && file.name !== '' ? file.name : id,
|
|
46
|
+
...(mime != null && !native ? { mime } : {}), ...(Number.isFinite(size) ? { size } : {}),
|
|
47
|
+
...(resourceKey != null && resourceKey !== '' ? { resourceKey } : {}),
|
|
48
|
+
}
|
|
49
|
+
}
|
|
50
|
+
|
|
51
|
+
const list: DriveLister['list'] = async (folder, limit) => {
|
|
52
|
+
const items: DriveItem[] = []
|
|
53
|
+
let token: string | undefined
|
|
54
|
+
let unsupported = 0
|
|
55
|
+
let files = 0
|
|
56
|
+
for (let page = 0; page < DRIVE_API_MAX_PAGES; page++) {
|
|
57
|
+
const answer = await json<DriveApiList>(urls.apiListUrl(folder.id, token), headersOf(folder.id, folder.resourceKey))
|
|
58
|
+
for (const file of answer.files ?? []) {
|
|
59
|
+
const item = itemOf(file)
|
|
60
|
+
if (item == null) {
|
|
61
|
+
unsupported++
|
|
62
|
+
continue
|
|
63
|
+
}
|
|
64
|
+
items.push(item)
|
|
65
|
+
if (item.kind !== GoogleDriveKind.Folder) files++
|
|
66
|
+
}
|
|
67
|
+
token = answer.nextPageToken
|
|
68
|
+
if (token == null || token === '' || files >= limit) break
|
|
69
|
+
}
|
|
70
|
+
log.debug('Listed a folder through the Drive API', { folder: folder.id, items: items.length, unsupported, more: token != null })
|
|
71
|
+
return items
|
|
72
|
+
}
|
|
73
|
+
|
|
74
|
+
const resolve: DriveLister['resolve'] = async (_url, link) => {
|
|
75
|
+
const item = itemOf(await json<DriveApiFile>(urls.apiFileUrl(link.id), headersOf(link.id, link.resourceKey)))
|
|
76
|
+
if (item == null) throw new GoogleDriveError(`${GoogleDriveFailure.Unrecognized}:item-type`)
|
|
77
|
+
return { ...item }
|
|
78
|
+
}
|
|
79
|
+
|
|
80
|
+
return { list, resolve }
|
|
81
|
+
}
|
|
@@ -0,0 +1,52 @@
|
|
|
1
|
+
import { GoogleDriveKind } from '../consts.js'
|
|
2
|
+
|
|
3
|
+
/** The log scope of the package. */
|
|
4
|
+
export const LOG_SCOPE = 'llm-reader-google-drive'
|
|
5
|
+
|
|
6
|
+
/** File downloads probed at once. */
|
|
7
|
+
export const DRIVE_PROBE_CONCURRENCY = 4
|
|
8
|
+
|
|
9
|
+
/** Requests one probe may take: the download, then Drive's warning form (and its legacy link). */
|
|
10
|
+
export const DRIVE_PROBE_STEPS = 3
|
|
11
|
+
|
|
12
|
+
/** Bytes of a probe's answer read — an interstitial or refusal page. */
|
|
13
|
+
export const DRIVE_PAGE_BYTES = 256 * 1024
|
|
14
|
+
|
|
15
|
+
/** Bytes of one folder listing (a page of the embedded view, a Drive API page). */
|
|
16
|
+
export const DRIVE_LISTING_BYTES = 4 * 1024 * 1024
|
|
17
|
+
|
|
18
|
+
/** Drive API pages read per folder at most. */
|
|
19
|
+
export const DRIVE_API_MAX_PAGES = 20
|
|
20
|
+
|
|
21
|
+
/**
|
|
22
|
+
* Drive serves files below this size without its "can't scan for viruses" form; an entry whose size
|
|
23
|
+
* is known and below it is not probed.
|
|
24
|
+
*/
|
|
25
|
+
export const DRIVE_SCAN_LIMIT_BYTES = 100 * 1024 * 1024
|
|
26
|
+
|
|
27
|
+
/** The media type prefix of Google's own item types — none has bytes of its own. */
|
|
28
|
+
export const GOOGLE_APPS_PREFIX = 'application/vnd.google-apps.'
|
|
29
|
+
|
|
30
|
+
/** The shortcut type; its target is listed in its place. */
|
|
31
|
+
export const GOOGLE_APPS_SHORTCUT = 'application/vnd.google-apps.shortcut'
|
|
32
|
+
|
|
33
|
+
/** The Google item types the source reads; every other `application/vnd.google-apps.*` is left out. */
|
|
34
|
+
export const GOOGLE_APPS_KINDS: Readonly<Record<string, GoogleDriveKind>> = Object.freeze({
|
|
35
|
+
'application/vnd.google-apps.folder': GoogleDriveKind.Folder,
|
|
36
|
+
'application/vnd.google-apps.document': GoogleDriveKind.Document,
|
|
37
|
+
'application/vnd.google-apps.spreadsheet': GoogleDriveKind.Spreadsheet,
|
|
38
|
+
'application/vnd.google-apps.presentation': GoogleDriveKind.Presentation,
|
|
39
|
+
'application/vnd.google-apps.drawing': GoogleDriveKind.Drawing,
|
|
40
|
+
})
|
|
41
|
+
|
|
42
|
+
/** A refusal that means an exhausted download or API quota. */
|
|
43
|
+
export const QUOTA_PATTERN = /too many users have viewed or downloaded|quota|dailyLimitExceeded|userRateLimitExceeded|rateLimitExceeded|RESOURCE_EXHAUSTED/i
|
|
44
|
+
|
|
45
|
+
/** A Drive API refusal of the key itself. */
|
|
46
|
+
export const API_KEY_PATTERN = /API_KEY_INVALID|API key not valid|keyInvalid|keyExpired|accessNotConfigured|SERVICE_DISABLED|API_KEY_SERVICE_BLOCKED/
|
|
47
|
+
|
|
48
|
+
/** A page that asks to sign in or to request access. */
|
|
49
|
+
export const NOT_PUBLIC_PATTERN = /ServiceLogin|\/v3\/signin|you need access|request access|need permission/i
|
|
50
|
+
|
|
51
|
+
/** A refusal raised by a file-streams Drive resolver inside an injected fetch. */
|
|
52
|
+
export const RESOLVER_PATTERN = /resolver:google-drive/
|
|
@@ -0,0 +1,83 @@
|
|
|
1
|
+
import { logger } from '@owlmeans/log'
|
|
2
|
+
import { parseHTML } from 'linkedom'
|
|
3
|
+
import { GoogleDriveFailure, GoogleDriveKind } from '../consts.js'
|
|
4
|
+
import { DRIVE_ID_PATTERN } from '../consts.local.js'
|
|
5
|
+
import { GoogleDriveError } from '../errors.js'
|
|
6
|
+
import type { GoogleDriveUrlHelper } from '../types.js'
|
|
7
|
+
import { DRIVE_LISTING_BYTES, GOOGLE_APPS_KINDS, GOOGLE_APPS_PREFIX, LOG_SCOPE } from './consts.local.js'
|
|
8
|
+
import type { DriveHttp, DriveItem, DriveLister } from './types.local.js'
|
|
9
|
+
|
|
10
|
+
const log = logger(LOG_SCOPE)
|
|
11
|
+
|
|
12
|
+
/**
|
|
13
|
+
* Folders listed without a key, from Drive's static embedded folder view
|
|
14
|
+
* (`/embeddedfolderview?id=<id>`): every `div.flip-entry` (`id="entry-<fileId>"`) carries a link
|
|
15
|
+
* (`/file/d/<id>/view` for a file, `/drive/folders/<id>` for a sub-folder, `docs.google.com/…` for an
|
|
16
|
+
* editor file), its `.flip-entry-title` and a type icon (`…/type/<media type>`). No sizes.
|
|
17
|
+
*/
|
|
18
|
+
export const makeEmbeddedLister = (http: DriveHttp, urls: GoogleDriveUrlHelper): DriveLister => {
|
|
19
|
+
const mimeOf = (src: string | null | undefined, base: string): string | undefined => {
|
|
20
|
+
if (src == null || src === '') return undefined
|
|
21
|
+
try {
|
|
22
|
+
const type = /\/type\/(.+)$/.exec(new URL(src, base).pathname)?.[1]
|
|
23
|
+
return type != null ? decodeURIComponent(type) : undefined
|
|
24
|
+
} catch {
|
|
25
|
+
return undefined
|
|
26
|
+
}
|
|
27
|
+
}
|
|
28
|
+
|
|
29
|
+
const list: DriveLister['list'] = async folder => {
|
|
30
|
+
const response = await http.get(urls.folderViewUrl(folder.id, folder.resourceKey), { maxBytes: DRIVE_LISTING_BYTES })
|
|
31
|
+
const refused = await http.refusal(response)
|
|
32
|
+
if (refused != null) throw refused
|
|
33
|
+
const base = response.url !== '' ? response.url : urls.folderViewUrl(folder.id)
|
|
34
|
+
const page = await response.text()
|
|
35
|
+
const { document } = parseHTML(page)
|
|
36
|
+
if (document.querySelector('#flip-contents, .flip-entries, .flip-entry') == null) {
|
|
37
|
+
throw http.pageFailure(page) ?? new GoogleDriveError(`${GoogleDriveFailure.Unrecognized}:folder-view`)
|
|
38
|
+
}
|
|
39
|
+
const items: DriveItem[] = []
|
|
40
|
+
let unsupported = 0
|
|
41
|
+
for (const entry of document.querySelectorAll('.flip-entry')) {
|
|
42
|
+
const href = entry.querySelector('a[href]')?.getAttribute('href')
|
|
43
|
+
let link
|
|
44
|
+
try {
|
|
45
|
+
link = href != null ? urls.parse(new URL(href, base).href) : undefined
|
|
46
|
+
} catch {
|
|
47
|
+
link = undefined
|
|
48
|
+
}
|
|
49
|
+
const mime = mimeOf(entry.querySelector('.flip-entry-list-icon img, .flip-entry-icon img')?.getAttribute('src'), base)
|
|
50
|
+
const fallback = /^entry-(.+)$/.exec(entry.getAttribute('id') ?? '')?.[1]
|
|
51
|
+
const id = link?.id ?? (fallback != null && DRIVE_ID_PATTERN.test(fallback) ? fallback : undefined)
|
|
52
|
+
const kind = link != null && link.kind !== GoogleDriveKind.Unresolved ? link.kind
|
|
53
|
+
: mime != null && mime.startsWith(GOOGLE_APPS_PREFIX) ? GOOGLE_APPS_KINDS[mime]
|
|
54
|
+
: id != null ? GoogleDriveKind.File : undefined
|
|
55
|
+
if (id == null || kind == null) {
|
|
56
|
+
unsupported++
|
|
57
|
+
continue
|
|
58
|
+
}
|
|
59
|
+
const title = entry.querySelector('.flip-entry-title')?.textContent?.trim()
|
|
60
|
+
items.push({
|
|
61
|
+
id, kind, name: title != null && title !== '' ? title : id,
|
|
62
|
+
...(mime != null && !mime.startsWith(GOOGLE_APPS_PREFIX) ? { mime } : {}),
|
|
63
|
+
...(link?.resourceKey != null ? { resourceKey: link.resourceKey } : {}),
|
|
64
|
+
})
|
|
65
|
+
}
|
|
66
|
+
log.debug('Listed a folder', { folder: folder.id, items: items.length, unsupported })
|
|
67
|
+
return items
|
|
68
|
+
}
|
|
69
|
+
|
|
70
|
+
const resolve: DriveLister['resolve'] = async (url, link) => {
|
|
71
|
+
// `open?id=` answers with a redirect to the item's own page; only where it lands is read.
|
|
72
|
+
const response = await http.get(url, { maxBytes: DRIVE_LISTING_BYTES })
|
|
73
|
+
const refused = await http.refusal(response)
|
|
74
|
+
if (refused != null) throw refused
|
|
75
|
+
await http.discard(response)
|
|
76
|
+
const landed = response.url !== '' ? urls.parse(response.url) : undefined
|
|
77
|
+
const resourceKey = landed?.resourceKey ?? link.resourceKey
|
|
78
|
+
const kind = landed != null && landed.kind !== GoogleDriveKind.Unresolved ? landed.kind : GoogleDriveKind.File
|
|
79
|
+
return { kind, id: landed?.id ?? link.id, ...(resourceKey != null ? { resourceKey } : {}) }
|
|
80
|
+
}
|
|
81
|
+
|
|
82
|
+
return { list, resolve }
|
|
83
|
+
}
|
|
@@ -0,0 +1,66 @@
|
|
|
1
|
+
import type { ReaderSourceContext } from '@owlmeans/llm-reader'
|
|
2
|
+
import { GoogleDriveFailure } from '../consts.js'
|
|
3
|
+
import { GoogleDriveError } from '../errors.js'
|
|
4
|
+
import type { GoogleDriveUrlHelper } from '../types.js'
|
|
5
|
+
import { API_KEY_PATTERN, NOT_PUBLIC_PATTERN, QUOTA_PATTERN, RESOLVER_PATTERN } from './consts.local.js'
|
|
6
|
+
import type { DriveHttp } from './types.local.js'
|
|
7
|
+
|
|
8
|
+
/** Google, asked through one collect's guarded fetch — its answers read as Drive refusals. */
|
|
9
|
+
export const makeDriveHttp = (context: ReaderSourceContext, urls: GoogleDriveUrlHelper): DriveHttp => {
|
|
10
|
+
const fail = (reason: string): GoogleDriveError => new GoogleDriveError(reason)
|
|
11
|
+
|
|
12
|
+
const messagesOf = (error: unknown): string => {
|
|
13
|
+
const parts: string[] = []
|
|
14
|
+
for (let current = error, depth = 0; current != null && depth < 4; current = (current as { cause?: unknown }).cause, depth++) {
|
|
15
|
+
parts.push(String((current as { message?: unknown }).message ?? current))
|
|
16
|
+
}
|
|
17
|
+
return parts.join(' ')
|
|
18
|
+
}
|
|
19
|
+
|
|
20
|
+
const get: DriveHttp['get'] = async (url, init) => {
|
|
21
|
+
try {
|
|
22
|
+
return await context.fetch(url, init)
|
|
23
|
+
} catch (error) {
|
|
24
|
+
if (context.signal?.aborted !== true && RESOLVER_PATTERN.test(messagesOf(error))) {
|
|
25
|
+
throw Object.assign(fail(GoogleDriveFailure.Resolver), { cause: error, url })
|
|
26
|
+
}
|
|
27
|
+
throw error
|
|
28
|
+
}
|
|
29
|
+
}
|
|
30
|
+
|
|
31
|
+
const discard: DriveHttp['discard'] = async response => {
|
|
32
|
+
await response.body?.cancel().catch(() => undefined)
|
|
33
|
+
}
|
|
34
|
+
|
|
35
|
+
const text: DriveHttp['text'] = async response => {
|
|
36
|
+
try {
|
|
37
|
+
return await response.text()
|
|
38
|
+
} catch {
|
|
39
|
+
return ''
|
|
40
|
+
}
|
|
41
|
+
}
|
|
42
|
+
|
|
43
|
+
const pageFailure: DriveHttp['pageFailure'] = page => {
|
|
44
|
+
if (QUOTA_PATTERN.test(page)) return fail(GoogleDriveFailure.Quota)
|
|
45
|
+
if (NOT_PUBLIC_PATTERN.test(page)) return fail(GoogleDriveFailure.NotPublic)
|
|
46
|
+
return undefined
|
|
47
|
+
}
|
|
48
|
+
|
|
49
|
+
const refusal: DriveHttp['refusal'] = async response => {
|
|
50
|
+
if (response.url !== '' && urls.isSignIn(response.url)) {
|
|
51
|
+
await discard(response)
|
|
52
|
+
return fail(GoogleDriveFailure.NotPublic)
|
|
53
|
+
}
|
|
54
|
+
if (response.status < 400) return undefined
|
|
55
|
+
const body = await text(response)
|
|
56
|
+
const error = response.status === 429 || QUOTA_PATTERN.test(body) ? fail(GoogleDriveFailure.Quota)
|
|
57
|
+
: API_KEY_PATTERN.test(body) ? fail(GoogleDriveFailure.ApiKey)
|
|
58
|
+
: response.status === 404 ? fail(GoogleDriveFailure.NotFound)
|
|
59
|
+
: response.status === 401 || response.status === 403 ? fail(GoogleDriveFailure.NotPublic)
|
|
60
|
+
: fail(`${GoogleDriveFailure.Status}:${response.status}`)
|
|
61
|
+
error.status = response.status
|
|
62
|
+
return error
|
|
63
|
+
}
|
|
64
|
+
|
|
65
|
+
return { get, refusal, pageFailure, text, discard }
|
|
66
|
+
}
|
|
@@ -0,0 +1,84 @@
|
|
|
1
|
+
import { parseHTML } from 'linkedom'
|
|
2
|
+
import { GoogleDriveFailure } from '../consts.js'
|
|
3
|
+
import { GoogleDriveError } from '../errors.js'
|
|
4
|
+
import type { GoogleDriveUrlHelper } from '../types.js'
|
|
5
|
+
import { nameUtils } from '../utils/names.js'
|
|
6
|
+
import { DRIVE_PAGE_BYTES, DRIVE_PROBE_STEPS } from './consts.local.js'
|
|
7
|
+
import type { DriveHttp, DriveProbe, ProbedFile } from './types.local.js'
|
|
8
|
+
|
|
9
|
+
/**
|
|
10
|
+
* Downloads asked before the collector takes them: one ranged GET (`bytes=0-0`) tells the name
|
|
11
|
+
* (`Content-Disposition`) and size (`Content-Range`) of a file Drive serves as an attachment; an HTML
|
|
12
|
+
* answer is Drive's "can't scan for viruses" form — followed by its action and hidden inputs, like a
|
|
13
|
+
* browser would — or a refusal (sign-in, quota).
|
|
14
|
+
*/
|
|
15
|
+
export const makeDriveProbe = (http: DriveHttp, urls: GoogleDriveUrlHelper): DriveProbe => {
|
|
16
|
+
const isAttachment = (response: Response): boolean =>
|
|
17
|
+
/attachment/i.test(response.headers.get('content-disposition') ?? '') || !/text\/html/i.test(response.headers.get('content-type') ?? '')
|
|
18
|
+
|
|
19
|
+
const sizeOf = (response: Response): number | undefined => {
|
|
20
|
+
const total = Number(/\/(\d+)\s*$/.exec(response.headers.get('content-range') ?? '')?.[1] ?? NaN)
|
|
21
|
+
if (Number.isFinite(total)) return total
|
|
22
|
+
const length = Number(response.headers.get('content-length') ?? NaN)
|
|
23
|
+
return response.status === 200 && Number.isFinite(length) ? length : undefined
|
|
24
|
+
}
|
|
25
|
+
|
|
26
|
+
const described = (url: string, response: Response, size = true): ProbedFile => {
|
|
27
|
+
const name = nameUtils.fromDisposition(response.headers.get('content-disposition'))
|
|
28
|
+
const bytes = size ? sizeOf(response) : undefined
|
|
29
|
+
return { url, ...(name != null ? { name } : {}), ...(bytes != null ? { size: bytes } : {}) }
|
|
30
|
+
}
|
|
31
|
+
|
|
32
|
+
/** The URL Drive's warning page sends a browser on to — its form with the hidden inputs, or the legacy confirm link. */
|
|
33
|
+
const followUp = (page: string, base: string): string | undefined => {
|
|
34
|
+
const { document } = parseHTML(page)
|
|
35
|
+
const form = document.querySelector('form#download-form') ?? document.querySelector('form[action*="download"]')
|
|
36
|
+
const action = form?.getAttribute('action')
|
|
37
|
+
if (form != null && action != null && action !== '') {
|
|
38
|
+
const next = new URL(action, base)
|
|
39
|
+
for (const input of form.querySelectorAll('input[type="hidden"][name]')) {
|
|
40
|
+
next.searchParams.set(input.getAttribute('name')!, input.getAttribute('value') ?? '')
|
|
41
|
+
}
|
|
42
|
+
if (!urls.owns(next.href)) throw new GoogleDriveError(`${GoogleDriveFailure.Unrecognized}:foreign-form`)
|
|
43
|
+
return next.href
|
|
44
|
+
}
|
|
45
|
+
const legacy = document.querySelector('a[href*="export=download"][href*="confirm="]')?.getAttribute('href')
|
|
46
|
+
if (legacy != null) {
|
|
47
|
+
const next = new URL(legacy, base)
|
|
48
|
+
return urls.owns(next.href) ? next.href : undefined
|
|
49
|
+
}
|
|
50
|
+
return undefined
|
|
51
|
+
}
|
|
52
|
+
|
|
53
|
+
const file: DriveProbe['file'] = async (id, resourceKey) => {
|
|
54
|
+
let url = urls.downloadUrl(id, resourceKey)
|
|
55
|
+
for (let step = 0; step < DRIVE_PROBE_STEPS; step++) {
|
|
56
|
+
const response = await http.get(url, { headers: { range: 'bytes=0-0' }, maxBytes: DRIVE_PAGE_BYTES })
|
|
57
|
+
const refused = await http.refusal(response)
|
|
58
|
+
if (refused != null) throw refused
|
|
59
|
+
if (isAttachment(response)) {
|
|
60
|
+
await http.discard(response)
|
|
61
|
+
return described(url, response)
|
|
62
|
+
}
|
|
63
|
+
const page = await http.text(response)
|
|
64
|
+
const next = followUp(page, response.url !== '' ? response.url : url)
|
|
65
|
+
if (next == null) throw http.pageFailure(page) ?? new GoogleDriveError(GoogleDriveFailure.Unrecognized)
|
|
66
|
+
url = next
|
|
67
|
+
}
|
|
68
|
+
throw new GoogleDriveError(`${GoogleDriveFailure.Unrecognized}:steps`)
|
|
69
|
+
}
|
|
70
|
+
|
|
71
|
+
const exported: DriveProbe['exported'] = async url => {
|
|
72
|
+
const response = await http.get(url, { maxBytes: DRIVE_PAGE_BYTES })
|
|
73
|
+
const refused = await http.refusal(response)
|
|
74
|
+
if (refused != null) throw refused
|
|
75
|
+
if (isAttachment(response)) {
|
|
76
|
+
await http.discard(response)
|
|
77
|
+
// An export is generated on the fly: its length (if any) is not worth admitting it by.
|
|
78
|
+
return described(url, response, false)
|
|
79
|
+
}
|
|
80
|
+
throw http.pageFailure(await http.text(response)) ?? new GoogleDriveError(GoogleDriveFailure.Unrecognized)
|
|
81
|
+
}
|
|
82
|
+
|
|
83
|
+
return { file, exported }
|
|
84
|
+
}
|
|
@@ -0,0 +1,91 @@
|
|
|
1
|
+
import type { SourceFetchInit } from '@owlmeans/llm-reader'
|
|
2
|
+
import type { GoogleDriveKind } from '../consts.js'
|
|
3
|
+
import type { GoogleDriveError } from '../errors.js'
|
|
4
|
+
import type { GoogleDriveLink } from '../types.js'
|
|
5
|
+
|
|
6
|
+
/** One folder to list, where its files go and how deep it is. */
|
|
7
|
+
export interface DriveFolder {
|
|
8
|
+
id: string
|
|
9
|
+
resourceKey?: string
|
|
10
|
+
/** The folder's path inside the linked folder, `''` or ending in `/`. */
|
|
11
|
+
prefix: string
|
|
12
|
+
/** `0` for the linked folder. */
|
|
13
|
+
level: number
|
|
14
|
+
}
|
|
15
|
+
|
|
16
|
+
/** A folder's child as a listing names it. */
|
|
17
|
+
export interface DriveItem {
|
|
18
|
+
id: string
|
|
19
|
+
kind: GoogleDriveKind
|
|
20
|
+
name: string
|
|
21
|
+
/** Its real media type — never a Google item type. */
|
|
22
|
+
mime?: string
|
|
23
|
+
size?: number
|
|
24
|
+
resourceKey?: string
|
|
25
|
+
}
|
|
26
|
+
|
|
27
|
+
/** A link with what Drive told about it. */
|
|
28
|
+
export interface ResolvedLink extends GoogleDriveLink {
|
|
29
|
+
name?: string
|
|
30
|
+
mime?: string
|
|
31
|
+
size?: number
|
|
32
|
+
}
|
|
33
|
+
|
|
34
|
+
/** A file's bytes: the URL a plain GET downloads, with what Drive declared. */
|
|
35
|
+
export interface ProbedFile {
|
|
36
|
+
url: string
|
|
37
|
+
name?: string
|
|
38
|
+
size?: number
|
|
39
|
+
}
|
|
40
|
+
|
|
41
|
+
/** One way of listing Drive folders — the keyless embedded view, or Drive API v3. */
|
|
42
|
+
export interface DriveLister {
|
|
43
|
+
/** A folder's children, up to about `limit`. Throws `GoogleDriveError` when Drive refuses. */
|
|
44
|
+
list: (folder: DriveFolder, limit: number) => Promise<DriveItem[]>
|
|
45
|
+
/** What an `open?id=` link points at. */
|
|
46
|
+
resolve: (url: string, link: GoogleDriveLink) => Promise<ResolvedLink>
|
|
47
|
+
}
|
|
48
|
+
|
|
49
|
+
/** Downloads asked before they are handed to the collector. */
|
|
50
|
+
export interface DriveProbe {
|
|
51
|
+
/**
|
|
52
|
+
* A stored file's download followed through Drive's warning form: a URL that serves the bytes to a
|
|
53
|
+
* plain GET, its name and size. Throws `GoogleDriveError` for a private, missing or quota-bound file.
|
|
54
|
+
*/
|
|
55
|
+
file: (id: string, resourceKey?: string) => Promise<ProbedFile>
|
|
56
|
+
/** An editor file's export, asked for its name (its body is dropped). */
|
|
57
|
+
exported: (url: string) => Promise<ProbedFile>
|
|
58
|
+
}
|
|
59
|
+
|
|
60
|
+
/** Requests to Google and the reading of its refusals. */
|
|
61
|
+
export interface DriveHttp {
|
|
62
|
+
/** A GET through the reader's guarded fetch; a file-streams Drive resolver's refusal becomes `GoogleDriveError('resolver')`. */
|
|
63
|
+
get: (url: string, init?: SourceFetchInit) => Promise<Response>
|
|
64
|
+
/**
|
|
65
|
+
* The refusal an answer stands for — Google's sign-in host, 401/403/404/429, quota and API-key
|
|
66
|
+
* errors, any other status ≥ 400 — or `undefined` for a usable answer. Reads the body of a refusal only.
|
|
67
|
+
*/
|
|
68
|
+
refusal: (response: Response) => Promise<GoogleDriveError | undefined>
|
|
69
|
+
/** What an HTML page served instead of the expected answer says: quota, sign-in, no access. */
|
|
70
|
+
pageFailure: (page: string) => GoogleDriveError | undefined
|
|
71
|
+
/** A body read as text; `''` when it cannot be read. */
|
|
72
|
+
text: (response: Response) => Promise<string>
|
|
73
|
+
/** Drop a body that is not read. */
|
|
74
|
+
discard: (response: Response) => Promise<void>
|
|
75
|
+
}
|
|
76
|
+
|
|
77
|
+
/** A Drive API v3 file resource — the fields asked for. */
|
|
78
|
+
export interface DriveApiFile {
|
|
79
|
+
id?: string
|
|
80
|
+
name?: string
|
|
81
|
+
mimeType?: string
|
|
82
|
+
size?: string
|
|
83
|
+
resourceKey?: string
|
|
84
|
+
shortcutDetails?: { targetId?: string, targetMimeType?: string, targetResourceKey?: string }
|
|
85
|
+
}
|
|
86
|
+
|
|
87
|
+
/** A Drive API v3 `files.list` page. */
|
|
88
|
+
export interface DriveApiList {
|
|
89
|
+
nextPageToken?: string
|
|
90
|
+
files?: DriveApiFile[]
|
|
91
|
+
}
|