@owlmeans/llm-reader-google-drive 0.0.0-stage → 0.2.1-alpha.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (90) hide show
  1. package/README.md +113 -2
  2. package/agent-meta/manifest.json +16 -0
  3. package/agent-meta/skills/llm-reader-google-drive/SKILL.md +131 -0
  4. package/build/consts.d.ts +45 -0
  5. package/build/consts.d.ts.map +1 -0
  6. package/build/consts.js +54 -0
  7. package/build/consts.js.map +1 -0
  8. package/build/consts.local.d.ts +3 -0
  9. package/build/consts.local.d.ts.map +1 -0
  10. package/build/consts.local.js +3 -0
  11. package/build/consts.local.js.map +1 -0
  12. package/build/errors.d.ts +14 -0
  13. package/build/errors.d.ts.map +1 -0
  14. package/build/errors.js +26 -0
  15. package/build/errors.js.map +1 -0
  16. package/build/index.d.ts +6 -0
  17. package/build/index.d.ts.map +1 -0
  18. package/build/index.js +5 -0
  19. package/build/index.js.map +1 -0
  20. package/build/source/api.d.ts +9 -0
  21. package/build/source/api.d.ts.map +1 -0
  22. package/build/source/api.js +80 -0
  23. package/build/source/api.js.map +1 -0
  24. package/build/source/consts.local.d.ts +33 -0
  25. package/build/source/consts.local.d.ts.map +1 -0
  26. package/build/source/consts.local.js +39 -0
  27. package/build/source/consts.local.js.map +1 -0
  28. package/build/source/embedded.d.ts +10 -0
  29. package/build/source/embedded.d.ts.map +1 -0
  30. package/build/source/embedded.js +82 -0
  31. package/build/source/embedded.js.map +1 -0
  32. package/build/source/http.d.ts +6 -0
  33. package/build/source/http.d.ts.map +1 -0
  34. package/build/source/http.js +61 -0
  35. package/build/source/http.js.map +1 -0
  36. package/build/source/probe.d.ts +10 -0
  37. package/build/source/probe.d.ts.map +1 -0
  38. package/build/source/probe.js +80 -0
  39. package/build/source/probe.js.map +1 -0
  40. package/build/source/types.local.d.ts +87 -0
  41. package/build/source/types.local.d.ts.map +1 -0
  42. package/build/source/types.local.js +2 -0
  43. package/build/source/types.local.js.map +1 -0
  44. package/build/source.d.ts +22 -0
  45. package/build/source.d.ts.map +1 -0
  46. package/build/source.js +169 -0
  47. package/build/source.js.map +1 -0
  48. package/build/types.d.ts +79 -0
  49. package/build/types.d.ts.map +1 -0
  50. package/build/types.js +2 -0
  51. package/build/types.js.map +1 -0
  52. package/build/url/consts.local.d.ts +10 -0
  53. package/build/url/consts.local.d.ts.map +1 -0
  54. package/build/url/consts.local.js +15 -0
  55. package/build/url/consts.local.js.map +1 -0
  56. package/build/url.d.ts +8 -0
  57. package/build/url.d.ts.map +1 -0
  58. package/build/url.js +117 -0
  59. package/build/url.js.map +1 -0
  60. package/build/utils/names/types.d.ts +10 -0
  61. package/build/utils/names/types.d.ts.map +1 -0
  62. package/build/utils/names/types.js +2 -0
  63. package/build/utils/names/types.js.map +1 -0
  64. package/build/utils/names.d.ts +4 -0
  65. package/build/utils/names.d.ts.map +1 -0
  66. package/build/utils/names.js +31 -0
  67. package/build/utils/names.js.map +1 -0
  68. package/package.json +62 -4
  69. package/src/consts.local.ts +2 -0
  70. package/src/consts.ts +60 -0
  71. package/src/errors.ts +30 -0
  72. package/src/index.ts +7 -0
  73. package/src/source/api.ts +81 -0
  74. package/src/source/consts.local.ts +52 -0
  75. package/src/source/embedded.ts +83 -0
  76. package/src/source/http.ts +66 -0
  77. package/src/source/probe.ts +84 -0
  78. package/src/source/types.local.ts +91 -0
  79. package/src/source.ts +179 -0
  80. package/src/types.ts +86 -0
  81. package/src/url/consts.local.ts +19 -0
  82. package/src/url.ts +121 -0
  83. package/src/utils/names/types.ts +9 -0
  84. package/src/utils/names.ts +36 -0
  85. package/tests/context.ts +212 -0
  86. package/tests/fixtures.ts +88 -0
  87. package/tests/live.spec.ts +53 -0
  88. package/tests/source.spec.ts +189 -0
  89. package/tests/tsconfig.json +12 -0
  90. package/tsconfig.json +16 -0
package/src/consts.ts ADDED
@@ -0,0 +1,60 @@
1
+ import type { GoogleDriveOrigins, GoogleExportFormats } from './types.js'
2
+
3
+ /** The name the source expands links under (`CollectResult.source`). */
4
+ export const GOOGLE_DRIVE_SOURCE = 'google-drive'
5
+
6
+ /** Where Google answers; tests point these at a stand-in. */
7
+ export const DEFAULT_GOOGLE_DRIVE_ORIGINS: Readonly<Required<GoogleDriveOrigins>> = Object.freeze({
8
+ drive: 'https://drive.google.com',
9
+ usercontent: 'https://drive.usercontent.google.com',
10
+ docs: 'https://docs.google.com',
11
+ accounts: 'https://accounts.google.com',
12
+ api: 'https://www.googleapis.com',
13
+ })
14
+
15
+ /** The format each Google editor file is exported as unless `exports` names another. */
16
+ export const DEFAULT_GOOGLE_EXPORTS: Readonly<Required<GoogleExportFormats>> = Object.freeze({
17
+ document: 'docx', spreadsheet: 'xlsx', presentation: 'pptx', drawing: 'png',
18
+ })
19
+
20
+ /** Sub-folder levels listed below the linked folder (`0` = its own files only). */
21
+ export const DEFAULT_GOOGLE_DRIVE_DEPTH = 3
22
+
23
+ /** Entries one link expands to at most — under the collect's own `maxFiles + 1`. */
24
+ export const DEFAULT_GOOGLE_DRIVE_MAX_ENTRIES = 200
25
+
26
+ /** Folders one link lists at most (each is one request). */
27
+ export const DEFAULT_GOOGLE_DRIVE_MAX_FOLDERS = 50
28
+
29
+ /** What a Drive link points at. */
30
+ export enum GoogleDriveKind {
31
+ Folder = 'folder',
32
+ /** A stored file — bytes behind the usercontent download. */
33
+ File = 'file',
34
+ Document = 'document',
35
+ Spreadsheet = 'spreadsheet',
36
+ Presentation = 'presentation',
37
+ Drawing = 'drawing',
38
+ /** An `open?id=` link: Drive itself tells what it is. */
39
+ Unresolved = 'unresolved',
40
+ }
41
+
42
+ /** Why a link or an entry could not be expanded — the `reason` of a `GoogleDriveError`. */
43
+ export enum GoogleDriveFailure {
44
+ /** Shared privately: Google answered with its sign-in page, 401/403 or "you need access". */
45
+ NotPublic = 'not-public',
46
+ /** Download or API quota exhausted ("too many users have viewed or downloaded", 429, rate limits). */
47
+ Quota = 'quota',
48
+ /** Drive answered 404 — no such item, or one an API key may not see. */
49
+ NotFound = 'not-found',
50
+ /** The Drive API refused the key (invalid, or the Drive API not enabled for it). */
51
+ ApiKey = 'api-key',
52
+ /** The URL is not a Drive or Docs link this source reads. */
53
+ NotALink = 'not-a-link',
54
+ /** A Drive answer this source does not know how to read. */
55
+ Unrecognized = 'unrecognized',
56
+ /** A file-streams Drive resolver inside the injected fetch refused the folder page. */
57
+ Resolver = 'resolver',
58
+ /** Any other refusing status (`status:<code>`). */
59
+ Status = 'status',
60
+ }
package/src/errors.ts ADDED
@@ -0,0 +1,30 @@
1
+ import { ResilientError } from '@owlmeans/error'
2
+ import { ReaderFetchError } from '@owlmeans/llm-reader'
3
+ import { GOOGLE_DRIVE_SOURCE, GoogleDriveFailure } from './consts.js'
4
+
5
+ /**
6
+ * A Drive link or entry that could not be expanded — `llm-reader:fetch:google-drive:<reason>[:<detail>]`.
7
+ * A reader fetch error, so a collect reports it as the link's failed item with this message as the reason.
8
+ */
9
+ export class GoogleDriveError extends ReaderFetchError {
10
+ public static override typeName = `GoogleDrive${ReaderFetchError.typeName}`
11
+
12
+ /** Why, read back from the message. */
13
+ public reason: GoogleDriveFailure = GoogleDriveFailure.Unrecognized
14
+
15
+ constructor(message: string = GoogleDriveFailure.Unrecognized) {
16
+ super(`${GOOGLE_DRIVE_SOURCE}:${message}`)
17
+ this.type = GoogleDriveError.typeName
18
+ this.finalizeUnmarshal()
19
+ }
20
+
21
+ public override finalizeUnmarshal(): void {
22
+ super.finalizeUnmarshal()
23
+ const reason = new RegExp(`${GOOGLE_DRIVE_SOURCE}:([a-z-]+)`).exec(this.message)?.[1]
24
+ this.reason = Object.values(GoogleDriveFailure).includes(reason as GoogleDriveFailure)
25
+ ? reason as GoogleDriveFailure
26
+ : GoogleDriveFailure.Unrecognized
27
+ }
28
+ }
29
+
30
+ ResilientError.registerErrorClass(GoogleDriveError)
package/src/index.ts ADDED
@@ -0,0 +1,7 @@
1
+ export type * from './types.js'
2
+
3
+ export * from './consts.js'
4
+ export * from './errors.js'
5
+
6
+ export * from './url.js'
7
+ export * from './source.js'
@@ -0,0 +1,81 @@
1
+ import { logger } from '@owlmeans/log'
2
+ import { GoogleDriveFailure, GoogleDriveKind } from '../consts.js'
3
+ import { DRIVE_ID_PATTERN } from '../consts.local.js'
4
+ import { GoogleDriveError } from '../errors.js'
5
+ import type { GoogleDriveUrlHelper } from '../types.js'
6
+ import { DRIVE_API_MAX_PAGES, DRIVE_LISTING_BYTES, GOOGLE_APPS_KINDS, GOOGLE_APPS_PREFIX, GOOGLE_APPS_SHORTCUT, LOG_SCOPE } from './consts.local.js'
7
+ import type { DriveApiFile, DriveApiList, DriveHttp, DriveItem, DriveLister } from './types.local.js'
8
+
9
+ const log = logger(LOG_SCOPE)
10
+
11
+ /**
12
+ * Folders listed through Drive API v3 with an API key (the `X-Goog-Api-Key` header, never the URL):
13
+ * `files.list` pages of `'<id>' in parents and trashed=false`, with sizes and media types; a
14
+ * shortcut is listed as its target; Google item types other than folders and editor files are left out.
15
+ */
16
+ export const makeApiLister = (http: DriveHttp, urls: GoogleDriveUrlHelper, apiKey: string): DriveLister => {
17
+ const headersOf = (id: string, resourceKey?: string): Record<string, string> => ({
18
+ 'x-goog-api-key': apiKey,
19
+ accept: 'application/json',
20
+ ...(resourceKey != null ? { 'x-goog-drive-resource-keys': `${id}/${resourceKey}` } : {}),
21
+ })
22
+
23
+ const json = async <T>(url: string, headers: Record<string, string>): Promise<T> => {
24
+ const response = await http.get(url, { headers, maxBytes: DRIVE_LISTING_BYTES })
25
+ const refused = await http.refusal(response)
26
+ if (refused != null) throw refused
27
+ try {
28
+ return JSON.parse(await response.text()) as T
29
+ } catch {
30
+ throw new GoogleDriveError(`${GoogleDriveFailure.Unrecognized}:api-answer`)
31
+ }
32
+ }
33
+
34
+ const itemOf = (file: DriveApiFile): DriveItem | undefined => {
35
+ const shortcut = file.mimeType === GOOGLE_APPS_SHORTCUT ? file.shortcutDetails : undefined
36
+ const id = shortcut != null ? shortcut.targetId : file.id
37
+ const mime = shortcut != null ? shortcut.targetMimeType : file.mimeType
38
+ const resourceKey = shortcut != null ? shortcut.targetResourceKey : file.resourceKey
39
+ if (id == null || !DRIVE_ID_PATTERN.test(id)) return undefined
40
+ const native = mime != null && mime.startsWith(GOOGLE_APPS_PREFIX)
41
+ const kind = native ? GOOGLE_APPS_KINDS[mime] : GoogleDriveKind.File
42
+ if (kind == null) return undefined
43
+ const size = Number(file.size ?? NaN)
44
+ return {
45
+ id, kind, name: file.name != null && file.name !== '' ? file.name : id,
46
+ ...(mime != null && !native ? { mime } : {}), ...(Number.isFinite(size) ? { size } : {}),
47
+ ...(resourceKey != null && resourceKey !== '' ? { resourceKey } : {}),
48
+ }
49
+ }
50
+
51
+ const list: DriveLister['list'] = async (folder, limit) => {
52
+ const items: DriveItem[] = []
53
+ let token: string | undefined
54
+ let unsupported = 0
55
+ let files = 0
56
+ for (let page = 0; page < DRIVE_API_MAX_PAGES; page++) {
57
+ const answer = await json<DriveApiList>(urls.apiListUrl(folder.id, token), headersOf(folder.id, folder.resourceKey))
58
+ for (const file of answer.files ?? []) {
59
+ const item = itemOf(file)
60
+ if (item == null) {
61
+ unsupported++
62
+ continue
63
+ }
64
+ items.push(item)
65
+ if (item.kind !== GoogleDriveKind.Folder) files++
66
+ }
67
+ token = answer.nextPageToken
68
+ if (token == null || token === '' || files >= limit) break
69
+ }
70
+ log.debug('Listed a folder through the Drive API', { folder: folder.id, items: items.length, unsupported, more: token != null })
71
+ return items
72
+ }
73
+
74
+ const resolve: DriveLister['resolve'] = async (_url, link) => {
75
+ const item = itemOf(await json<DriveApiFile>(urls.apiFileUrl(link.id), headersOf(link.id, link.resourceKey)))
76
+ if (item == null) throw new GoogleDriveError(`${GoogleDriveFailure.Unrecognized}:item-type`)
77
+ return { ...item }
78
+ }
79
+
80
+ return { list, resolve }
81
+ }
@@ -0,0 +1,52 @@
1
+ import { GoogleDriveKind } from '../consts.js'
2
+
3
+ /** The log scope of the package. */
4
+ export const LOG_SCOPE = 'llm-reader-google-drive'
5
+
6
+ /** File downloads probed at once. */
7
+ export const DRIVE_PROBE_CONCURRENCY = 4
8
+
9
+ /** Requests one probe may take: the download, then Drive's warning form (and its legacy link). */
10
+ export const DRIVE_PROBE_STEPS = 3
11
+
12
+ /** Bytes of a probe's answer read — an interstitial or refusal page. */
13
+ export const DRIVE_PAGE_BYTES = 256 * 1024
14
+
15
+ /** Bytes of one folder listing (a page of the embedded view, a Drive API page). */
16
+ export const DRIVE_LISTING_BYTES = 4 * 1024 * 1024
17
+
18
+ /** Drive API pages read per folder at most. */
19
+ export const DRIVE_API_MAX_PAGES = 20
20
+
21
+ /**
22
+ * Drive serves files below this size without its "can't scan for viruses" form; an entry whose size
23
+ * is known and below it is not probed.
24
+ */
25
+ export const DRIVE_SCAN_LIMIT_BYTES = 100 * 1024 * 1024
26
+
27
+ /** The media type prefix of Google's own item types — none has bytes of its own. */
28
+ export const GOOGLE_APPS_PREFIX = 'application/vnd.google-apps.'
29
+
30
+ /** The shortcut type; its target is listed in its place. */
31
+ export const GOOGLE_APPS_SHORTCUT = 'application/vnd.google-apps.shortcut'
32
+
33
+ /** The Google item types the source reads; every other `application/vnd.google-apps.*` is left out. */
34
+ export const GOOGLE_APPS_KINDS: Readonly<Record<string, GoogleDriveKind>> = Object.freeze({
35
+ 'application/vnd.google-apps.folder': GoogleDriveKind.Folder,
36
+ 'application/vnd.google-apps.document': GoogleDriveKind.Document,
37
+ 'application/vnd.google-apps.spreadsheet': GoogleDriveKind.Spreadsheet,
38
+ 'application/vnd.google-apps.presentation': GoogleDriveKind.Presentation,
39
+ 'application/vnd.google-apps.drawing': GoogleDriveKind.Drawing,
40
+ })
41
+
42
+ /** A refusal that means an exhausted download or API quota. */
43
+ export const QUOTA_PATTERN = /too many users have viewed or downloaded|quota|dailyLimitExceeded|userRateLimitExceeded|rateLimitExceeded|RESOURCE_EXHAUSTED/i
44
+
45
+ /** A Drive API refusal of the key itself. */
46
+ export const API_KEY_PATTERN = /API_KEY_INVALID|API key not valid|keyInvalid|keyExpired|accessNotConfigured|SERVICE_DISABLED|API_KEY_SERVICE_BLOCKED/
47
+
48
+ /** A page that asks to sign in or to request access. */
49
+ export const NOT_PUBLIC_PATTERN = /ServiceLogin|\/v3\/signin|you need access|request access|need permission/i
50
+
51
+ /** A refusal raised by a file-streams Drive resolver inside an injected fetch. */
52
+ export const RESOLVER_PATTERN = /resolver:google-drive/
@@ -0,0 +1,83 @@
1
+ import { logger } from '@owlmeans/log'
2
+ import { parseHTML } from 'linkedom'
3
+ import { GoogleDriveFailure, GoogleDriveKind } from '../consts.js'
4
+ import { DRIVE_ID_PATTERN } from '../consts.local.js'
5
+ import { GoogleDriveError } from '../errors.js'
6
+ import type { GoogleDriveUrlHelper } from '../types.js'
7
+ import { DRIVE_LISTING_BYTES, GOOGLE_APPS_KINDS, GOOGLE_APPS_PREFIX, LOG_SCOPE } from './consts.local.js'
8
+ import type { DriveHttp, DriveItem, DriveLister } from './types.local.js'
9
+
10
+ const log = logger(LOG_SCOPE)
11
+
12
+ /**
13
+ * Folders listed without a key, from Drive's static embedded folder view
14
+ * (`/embeddedfolderview?id=<id>`): every `div.flip-entry` (`id="entry-<fileId>"`) carries a link
15
+ * (`/file/d/<id>/view` for a file, `/drive/folders/<id>` for a sub-folder, `docs.google.com/…` for an
16
+ * editor file), its `.flip-entry-title` and a type icon (`…/type/<media type>`). No sizes.
17
+ */
18
+ export const makeEmbeddedLister = (http: DriveHttp, urls: GoogleDriveUrlHelper): DriveLister => {
19
+ const mimeOf = (src: string | null | undefined, base: string): string | undefined => {
20
+ if (src == null || src === '') return undefined
21
+ try {
22
+ const type = /\/type\/(.+)$/.exec(new URL(src, base).pathname)?.[1]
23
+ return type != null ? decodeURIComponent(type) : undefined
24
+ } catch {
25
+ return undefined
26
+ }
27
+ }
28
+
29
+ const list: DriveLister['list'] = async folder => {
30
+ const response = await http.get(urls.folderViewUrl(folder.id, folder.resourceKey), { maxBytes: DRIVE_LISTING_BYTES })
31
+ const refused = await http.refusal(response)
32
+ if (refused != null) throw refused
33
+ const base = response.url !== '' ? response.url : urls.folderViewUrl(folder.id)
34
+ const page = await response.text()
35
+ const { document } = parseHTML(page)
36
+ if (document.querySelector('#flip-contents, .flip-entries, .flip-entry') == null) {
37
+ throw http.pageFailure(page) ?? new GoogleDriveError(`${GoogleDriveFailure.Unrecognized}:folder-view`)
38
+ }
39
+ const items: DriveItem[] = []
40
+ let unsupported = 0
41
+ for (const entry of document.querySelectorAll('.flip-entry')) {
42
+ const href = entry.querySelector('a[href]')?.getAttribute('href')
43
+ let link
44
+ try {
45
+ link = href != null ? urls.parse(new URL(href, base).href) : undefined
46
+ } catch {
47
+ link = undefined
48
+ }
49
+ const mime = mimeOf(entry.querySelector('.flip-entry-list-icon img, .flip-entry-icon img')?.getAttribute('src'), base)
50
+ const fallback = /^entry-(.+)$/.exec(entry.getAttribute('id') ?? '')?.[1]
51
+ const id = link?.id ?? (fallback != null && DRIVE_ID_PATTERN.test(fallback) ? fallback : undefined)
52
+ const kind = link != null && link.kind !== GoogleDriveKind.Unresolved ? link.kind
53
+ : mime != null && mime.startsWith(GOOGLE_APPS_PREFIX) ? GOOGLE_APPS_KINDS[mime]
54
+ : id != null ? GoogleDriveKind.File : undefined
55
+ if (id == null || kind == null) {
56
+ unsupported++
57
+ continue
58
+ }
59
+ const title = entry.querySelector('.flip-entry-title')?.textContent?.trim()
60
+ items.push({
61
+ id, kind, name: title != null && title !== '' ? title : id,
62
+ ...(mime != null && !mime.startsWith(GOOGLE_APPS_PREFIX) ? { mime } : {}),
63
+ ...(link?.resourceKey != null ? { resourceKey: link.resourceKey } : {}),
64
+ })
65
+ }
66
+ log.debug('Listed a folder', { folder: folder.id, items: items.length, unsupported })
67
+ return items
68
+ }
69
+
70
+ const resolve: DriveLister['resolve'] = async (url, link) => {
71
+ // `open?id=` answers with a redirect to the item's own page; only where it lands is read.
72
+ const response = await http.get(url, { maxBytes: DRIVE_LISTING_BYTES })
73
+ const refused = await http.refusal(response)
74
+ if (refused != null) throw refused
75
+ await http.discard(response)
76
+ const landed = response.url !== '' ? urls.parse(response.url) : undefined
77
+ const resourceKey = landed?.resourceKey ?? link.resourceKey
78
+ const kind = landed != null && landed.kind !== GoogleDriveKind.Unresolved ? landed.kind : GoogleDriveKind.File
79
+ return { kind, id: landed?.id ?? link.id, ...(resourceKey != null ? { resourceKey } : {}) }
80
+ }
81
+
82
+ return { list, resolve }
83
+ }
@@ -0,0 +1,66 @@
1
+ import type { ReaderSourceContext } from '@owlmeans/llm-reader'
2
+ import { GoogleDriveFailure } from '../consts.js'
3
+ import { GoogleDriveError } from '../errors.js'
4
+ import type { GoogleDriveUrlHelper } from '../types.js'
5
+ import { API_KEY_PATTERN, NOT_PUBLIC_PATTERN, QUOTA_PATTERN, RESOLVER_PATTERN } from './consts.local.js'
6
+ import type { DriveHttp } from './types.local.js'
7
+
8
+ /** Google, asked through one collect's guarded fetch — its answers read as Drive refusals. */
9
+ export const makeDriveHttp = (context: ReaderSourceContext, urls: GoogleDriveUrlHelper): DriveHttp => {
10
+ const fail = (reason: string): GoogleDriveError => new GoogleDriveError(reason)
11
+
12
+ const messagesOf = (error: unknown): string => {
13
+ const parts: string[] = []
14
+ for (let current = error, depth = 0; current != null && depth < 4; current = (current as { cause?: unknown }).cause, depth++) {
15
+ parts.push(String((current as { message?: unknown }).message ?? current))
16
+ }
17
+ return parts.join(' ')
18
+ }
19
+
20
+ const get: DriveHttp['get'] = async (url, init) => {
21
+ try {
22
+ return await context.fetch(url, init)
23
+ } catch (error) {
24
+ if (context.signal?.aborted !== true && RESOLVER_PATTERN.test(messagesOf(error))) {
25
+ throw Object.assign(fail(GoogleDriveFailure.Resolver), { cause: error, url })
26
+ }
27
+ throw error
28
+ }
29
+ }
30
+
31
+ const discard: DriveHttp['discard'] = async response => {
32
+ await response.body?.cancel().catch(() => undefined)
33
+ }
34
+
35
+ const text: DriveHttp['text'] = async response => {
36
+ try {
37
+ return await response.text()
38
+ } catch {
39
+ return ''
40
+ }
41
+ }
42
+
43
+ const pageFailure: DriveHttp['pageFailure'] = page => {
44
+ if (QUOTA_PATTERN.test(page)) return fail(GoogleDriveFailure.Quota)
45
+ if (NOT_PUBLIC_PATTERN.test(page)) return fail(GoogleDriveFailure.NotPublic)
46
+ return undefined
47
+ }
48
+
49
+ const refusal: DriveHttp['refusal'] = async response => {
50
+ if (response.url !== '' && urls.isSignIn(response.url)) {
51
+ await discard(response)
52
+ return fail(GoogleDriveFailure.NotPublic)
53
+ }
54
+ if (response.status < 400) return undefined
55
+ const body = await text(response)
56
+ const error = response.status === 429 || QUOTA_PATTERN.test(body) ? fail(GoogleDriveFailure.Quota)
57
+ : API_KEY_PATTERN.test(body) ? fail(GoogleDriveFailure.ApiKey)
58
+ : response.status === 404 ? fail(GoogleDriveFailure.NotFound)
59
+ : response.status === 401 || response.status === 403 ? fail(GoogleDriveFailure.NotPublic)
60
+ : fail(`${GoogleDriveFailure.Status}:${response.status}`)
61
+ error.status = response.status
62
+ return error
63
+ }
64
+
65
+ return { get, refusal, pageFailure, text, discard }
66
+ }
@@ -0,0 +1,84 @@
1
+ import { parseHTML } from 'linkedom'
2
+ import { GoogleDriveFailure } from '../consts.js'
3
+ import { GoogleDriveError } from '../errors.js'
4
+ import type { GoogleDriveUrlHelper } from '../types.js'
5
+ import { nameUtils } from '../utils/names.js'
6
+ import { DRIVE_PAGE_BYTES, DRIVE_PROBE_STEPS } from './consts.local.js'
7
+ import type { DriveHttp, DriveProbe, ProbedFile } from './types.local.js'
8
+
9
+ /**
10
+ * Downloads asked before the collector takes them: one ranged GET (`bytes=0-0`) tells the name
11
+ * (`Content-Disposition`) and size (`Content-Range`) of a file Drive serves as an attachment; an HTML
12
+ * answer is Drive's "can't scan for viruses" form — followed by its action and hidden inputs, like a
13
+ * browser would — or a refusal (sign-in, quota).
14
+ */
15
+ export const makeDriveProbe = (http: DriveHttp, urls: GoogleDriveUrlHelper): DriveProbe => {
16
+ const isAttachment = (response: Response): boolean =>
17
+ /attachment/i.test(response.headers.get('content-disposition') ?? '') || !/text\/html/i.test(response.headers.get('content-type') ?? '')
18
+
19
+ const sizeOf = (response: Response): number | undefined => {
20
+ const total = Number(/\/(\d+)\s*$/.exec(response.headers.get('content-range') ?? '')?.[1] ?? NaN)
21
+ if (Number.isFinite(total)) return total
22
+ const length = Number(response.headers.get('content-length') ?? NaN)
23
+ return response.status === 200 && Number.isFinite(length) ? length : undefined
24
+ }
25
+
26
+ const described = (url: string, response: Response, size = true): ProbedFile => {
27
+ const name = nameUtils.fromDisposition(response.headers.get('content-disposition'))
28
+ const bytes = size ? sizeOf(response) : undefined
29
+ return { url, ...(name != null ? { name } : {}), ...(bytes != null ? { size: bytes } : {}) }
30
+ }
31
+
32
+ /** The URL Drive's warning page sends a browser on to — its form with the hidden inputs, or the legacy confirm link. */
33
+ const followUp = (page: string, base: string): string | undefined => {
34
+ const { document } = parseHTML(page)
35
+ const form = document.querySelector('form#download-form') ?? document.querySelector('form[action*="download"]')
36
+ const action = form?.getAttribute('action')
37
+ if (form != null && action != null && action !== '') {
38
+ const next = new URL(action, base)
39
+ for (const input of form.querySelectorAll('input[type="hidden"][name]')) {
40
+ next.searchParams.set(input.getAttribute('name')!, input.getAttribute('value') ?? '')
41
+ }
42
+ if (!urls.owns(next.href)) throw new GoogleDriveError(`${GoogleDriveFailure.Unrecognized}:foreign-form`)
43
+ return next.href
44
+ }
45
+ const legacy = document.querySelector('a[href*="export=download"][href*="confirm="]')?.getAttribute('href')
46
+ if (legacy != null) {
47
+ const next = new URL(legacy, base)
48
+ return urls.owns(next.href) ? next.href : undefined
49
+ }
50
+ return undefined
51
+ }
52
+
53
+ const file: DriveProbe['file'] = async (id, resourceKey) => {
54
+ let url = urls.downloadUrl(id, resourceKey)
55
+ for (let step = 0; step < DRIVE_PROBE_STEPS; step++) {
56
+ const response = await http.get(url, { headers: { range: 'bytes=0-0' }, maxBytes: DRIVE_PAGE_BYTES })
57
+ const refused = await http.refusal(response)
58
+ if (refused != null) throw refused
59
+ if (isAttachment(response)) {
60
+ await http.discard(response)
61
+ return described(url, response)
62
+ }
63
+ const page = await http.text(response)
64
+ const next = followUp(page, response.url !== '' ? response.url : url)
65
+ if (next == null) throw http.pageFailure(page) ?? new GoogleDriveError(GoogleDriveFailure.Unrecognized)
66
+ url = next
67
+ }
68
+ throw new GoogleDriveError(`${GoogleDriveFailure.Unrecognized}:steps`)
69
+ }
70
+
71
+ const exported: DriveProbe['exported'] = async url => {
72
+ const response = await http.get(url, { maxBytes: DRIVE_PAGE_BYTES })
73
+ const refused = await http.refusal(response)
74
+ if (refused != null) throw refused
75
+ if (isAttachment(response)) {
76
+ await http.discard(response)
77
+ // An export is generated on the fly: its length (if any) is not worth admitting it by.
78
+ return described(url, response, false)
79
+ }
80
+ throw http.pageFailure(await http.text(response)) ?? new GoogleDriveError(GoogleDriveFailure.Unrecognized)
81
+ }
82
+
83
+ return { file, exported }
84
+ }
@@ -0,0 +1,91 @@
1
+ import type { SourceFetchInit } from '@owlmeans/llm-reader'
2
+ import type { GoogleDriveKind } from '../consts.js'
3
+ import type { GoogleDriveError } from '../errors.js'
4
+ import type { GoogleDriveLink } from '../types.js'
5
+
6
+ /** One folder to list, where its files go and how deep it is. */
7
+ export interface DriveFolder {
8
+ id: string
9
+ resourceKey?: string
10
+ /** The folder's path inside the linked folder, `''` or ending in `/`. */
11
+ prefix: string
12
+ /** `0` for the linked folder. */
13
+ level: number
14
+ }
15
+
16
+ /** A folder's child as a listing names it. */
17
+ export interface DriveItem {
18
+ id: string
19
+ kind: GoogleDriveKind
20
+ name: string
21
+ /** Its real media type — never a Google item type. */
22
+ mime?: string
23
+ size?: number
24
+ resourceKey?: string
25
+ }
26
+
27
+ /** A link with what Drive told about it. */
28
+ export interface ResolvedLink extends GoogleDriveLink {
29
+ name?: string
30
+ mime?: string
31
+ size?: number
32
+ }
33
+
34
+ /** A file's bytes: the URL a plain GET downloads, with what Drive declared. */
35
+ export interface ProbedFile {
36
+ url: string
37
+ name?: string
38
+ size?: number
39
+ }
40
+
41
+ /** One way of listing Drive folders — the keyless embedded view, or Drive API v3. */
42
+ export interface DriveLister {
43
+ /** A folder's children, up to about `limit`. Throws `GoogleDriveError` when Drive refuses. */
44
+ list: (folder: DriveFolder, limit: number) => Promise<DriveItem[]>
45
+ /** What an `open?id=` link points at. */
46
+ resolve: (url: string, link: GoogleDriveLink) => Promise<ResolvedLink>
47
+ }
48
+
49
+ /** Downloads asked before they are handed to the collector. */
50
+ export interface DriveProbe {
51
+ /**
52
+ * A stored file's download followed through Drive's warning form: a URL that serves the bytes to a
53
+ * plain GET, its name and size. Throws `GoogleDriveError` for a private, missing or quota-bound file.
54
+ */
55
+ file: (id: string, resourceKey?: string) => Promise<ProbedFile>
56
+ /** An editor file's export, asked for its name (its body is dropped). */
57
+ exported: (url: string) => Promise<ProbedFile>
58
+ }
59
+
60
+ /** Requests to Google and the reading of its refusals. */
61
+ export interface DriveHttp {
62
+ /** A GET through the reader's guarded fetch; a file-streams Drive resolver's refusal becomes `GoogleDriveError('resolver')`. */
63
+ get: (url: string, init?: SourceFetchInit) => Promise<Response>
64
+ /**
65
+ * The refusal an answer stands for — Google's sign-in host, 401/403/404/429, quota and API-key
66
+ * errors, any other status ≥ 400 — or `undefined` for a usable answer. Reads the body of a refusal only.
67
+ */
68
+ refusal: (response: Response) => Promise<GoogleDriveError | undefined>
69
+ /** What an HTML page served instead of the expected answer says: quota, sign-in, no access. */
70
+ pageFailure: (page: string) => GoogleDriveError | undefined
71
+ /** A body read as text; `''` when it cannot be read. */
72
+ text: (response: Response) => Promise<string>
73
+ /** Drop a body that is not read. */
74
+ discard: (response: Response) => Promise<void>
75
+ }
76
+
77
+ /** A Drive API v3 file resource — the fields asked for. */
78
+ export interface DriveApiFile {
79
+ id?: string
80
+ name?: string
81
+ mimeType?: string
82
+ size?: string
83
+ resourceKey?: string
84
+ shortcutDetails?: { targetId?: string, targetMimeType?: string, targetResourceKey?: string }
85
+ }
86
+
87
+ /** A Drive API v3 `files.list` page. */
88
+ export interface DriveApiList {
89
+ nextPageToken?: string
90
+ files?: DriveApiFile[]
91
+ }