@owlmeans/llm-reader-google-drive 0.0.0-stage → 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +113 -2
- package/agent-meta/manifest.json +16 -0
- package/agent-meta/skills/llm-reader-google-drive/SKILL.md +131 -0
- package/build/consts.d.ts +45 -0
- package/build/consts.d.ts.map +1 -0
- package/build/consts.js +54 -0
- package/build/consts.js.map +1 -0
- package/build/consts.local.d.ts +3 -0
- package/build/consts.local.d.ts.map +1 -0
- package/build/consts.local.js +3 -0
- package/build/consts.local.js.map +1 -0
- package/build/errors.d.ts +14 -0
- package/build/errors.d.ts.map +1 -0
- package/build/errors.js +26 -0
- package/build/errors.js.map +1 -0
- package/build/index.d.ts +6 -0
- package/build/index.d.ts.map +1 -0
- package/build/index.js +5 -0
- package/build/index.js.map +1 -0
- package/build/source/api.d.ts +9 -0
- package/build/source/api.d.ts.map +1 -0
- package/build/source/api.js +80 -0
- package/build/source/api.js.map +1 -0
- package/build/source/consts.local.d.ts +33 -0
- package/build/source/consts.local.d.ts.map +1 -0
- package/build/source/consts.local.js +39 -0
- package/build/source/consts.local.js.map +1 -0
- package/build/source/embedded.d.ts +10 -0
- package/build/source/embedded.d.ts.map +1 -0
- package/build/source/embedded.js +82 -0
- package/build/source/embedded.js.map +1 -0
- package/build/source/http.d.ts +6 -0
- package/build/source/http.d.ts.map +1 -0
- package/build/source/http.js +61 -0
- package/build/source/http.js.map +1 -0
- package/build/source/probe.d.ts +10 -0
- package/build/source/probe.d.ts.map +1 -0
- package/build/source/probe.js +80 -0
- package/build/source/probe.js.map +1 -0
- package/build/source/types.local.d.ts +87 -0
- package/build/source/types.local.d.ts.map +1 -0
- package/build/source/types.local.js +2 -0
- package/build/source/types.local.js.map +1 -0
- package/build/source.d.ts +22 -0
- package/build/source.d.ts.map +1 -0
- package/build/source.js +169 -0
- package/build/source.js.map +1 -0
- package/build/types.d.ts +79 -0
- package/build/types.d.ts.map +1 -0
- package/build/types.js +2 -0
- package/build/types.js.map +1 -0
- package/build/url/consts.local.d.ts +10 -0
- package/build/url/consts.local.d.ts.map +1 -0
- package/build/url/consts.local.js +15 -0
- package/build/url/consts.local.js.map +1 -0
- package/build/url.d.ts +8 -0
- package/build/url.d.ts.map +1 -0
- package/build/url.js +117 -0
- package/build/url.js.map +1 -0
- package/build/utils/names/types.d.ts +10 -0
- package/build/utils/names/types.d.ts.map +1 -0
- package/build/utils/names/types.js +2 -0
- package/build/utils/names/types.js.map +1 -0
- package/build/utils/names.d.ts +4 -0
- package/build/utils/names.d.ts.map +1 -0
- package/build/utils/names.js +31 -0
- package/build/utils/names.js.map +1 -0
- package/package.json +62 -4
- package/src/consts.local.ts +2 -0
- package/src/consts.ts +60 -0
- package/src/errors.ts +30 -0
- package/src/index.ts +7 -0
- package/src/source/api.ts +81 -0
- package/src/source/consts.local.ts +52 -0
- package/src/source/embedded.ts +83 -0
- package/src/source/http.ts +66 -0
- package/src/source/probe.ts +84 -0
- package/src/source/types.local.ts +91 -0
- package/src/source.ts +179 -0
- package/src/types.ts +86 -0
- package/src/url/consts.local.ts +19 -0
- package/src/url.ts +121 -0
- package/src/utils/names/types.ts +9 -0
- package/src/utils/names.ts +36 -0
- package/tests/context.ts +212 -0
- package/tests/fixtures.ts +88 -0
- package/tests/live.spec.ts +53 -0
- package/tests/source.spec.ts +189 -0
- package/tests/tsconfig.json +12 -0
- package/tsconfig.json +16 -0
package/README.md
CHANGED
|
@@ -1,3 +1,114 @@
|
|
|
1
|
-
#
|
|
1
|
+
# @owlmeans/llm-reader-google-drive
|
|
2
2
|
|
|
3
|
-
|
|
3
|
+
The Google Drive source of [`@owlmeans/llm-reader`](https://github.com/owlmeans/common/tree/main/packages/llm-reader):
|
|
4
|
+
a public Drive folder, file or Google Docs link given to `reader.collect` becomes the files behind it
|
|
5
|
+
— sub-folders included, editor files exported — downloaded by the reader under its guard and limits.
|
|
6
|
+
No Google account or API key is needed; an API key switches folder listing to Drive API v3.
|
|
7
|
+
|
|
8
|
+
## Installation
|
|
9
|
+
|
|
10
|
+
```sh
|
|
11
|
+
bun add @owlmeans/llm-reader-google-drive@^0.2.0 @owlmeans/llm-reader@^0.2.0
|
|
12
|
+
```
|
|
13
|
+
|
|
14
|
+
## Usage
|
|
15
|
+
|
|
16
|
+
```ts
|
|
17
|
+
import { createDirectoryCollectSink, createLlmReader } from '@owlmeans/llm-reader'
|
|
18
|
+
import { createGoogleDriveSource } from '@owlmeans/llm-reader-google-drive'
|
|
19
|
+
|
|
20
|
+
const reader = createLlmReader({ sources: [createGoogleDriveSource({ depth: 2, exports: { document: 'md' } })] })
|
|
21
|
+
const result = await reader.collect('https://drive.google.com/drive/folders/<id>?usp=share_link', {
|
|
22
|
+
sink: createDirectoryCollectSink(dir), prefix: 'links/01-drive',
|
|
23
|
+
})
|
|
24
|
+
// result.kind === 'source', result.source === 'google-drive'
|
|
25
|
+
// files at links/01-drive/<name>, links/01-drive/<sub-folder>/<name>, Docs as <title>.md
|
|
26
|
+
```
|
|
27
|
+
|
|
28
|
+
The reader has no source registry in a context: pass the source in `LlmReaderOptions.sources`
|
|
29
|
+
(`appendLlmReaderService(ctx, { sources: [createGoogleDriveSource()] })` in a context). Sources are
|
|
30
|
+
consulted in order, before the reader's built-in S3 source.
|
|
31
|
+
|
|
32
|
+
## What a link becomes
|
|
33
|
+
|
|
34
|
+
| Link | Entries |
|
|
35
|
+
|---|---|
|
|
36
|
+
| `drive.google.com/drive/folders/<id>` (`/drive/u/<n>/…`, `?usp=`, `?resourcekey=`), `folderview?id=` | every file, breadth-first down to `depth` sub-folder levels, at `<sub-folder>/<name>` |
|
|
37
|
+
| `drive.google.com/file/d/<id>/…`, `uc?id=`, `drive.usercontent.google.com/download?id=` | the file, named by Drive's `Content-Disposition` |
|
|
38
|
+
| `drive.google.com/open?id=<id>` | whatever Drive redirects it to (or the API reports) |
|
|
39
|
+
| `docs.google.com/{document,spreadsheets,presentation,drawings}/d/<id>/…` | the export: `docx`, `xlsx`, `pptx`, `png` unless `exports` says otherwise |
|
|
40
|
+
| any link that redirects onto one of these (a shortener) | the same, recognised from where it landed |
|
|
41
|
+
|
|
42
|
+
## Configuration
|
|
43
|
+
|
|
44
|
+
`createGoogleDriveSource(options?)`:
|
|
45
|
+
|
|
46
|
+
| Option | Default | Purpose |
|
|
47
|
+
|---|---|---|
|
|
48
|
+
| `apiKey` | — | A Google API key with the Drive API enabled: folders are listed through Drive API v3 (pages, sizes, media types, shortcuts). Sent as the `X-Goog-Api-Key` header, never in a URL. |
|
|
49
|
+
| `depth` | `DEFAULT_GOOGLE_DRIVE_DEPTH` (3) | Sub-folder levels listed below the linked folder; `0` lists its own files only. |
|
|
50
|
+
| `maxEntries` | `DEFAULT_GOOGLE_DRIVE_MAX_ENTRIES` (200) | Entries one link expands to; the collect's `maxFiles + 1` caps it too. |
|
|
51
|
+
| `maxFolders` | `DEFAULT_GOOGLE_DRIVE_MAX_FOLDERS` (50) | Folders one link lists — each is one request. |
|
|
52
|
+
| `exports` | `DEFAULT_GOOGLE_EXPORTS` | `document` (`docx`, `pdf`, `odt`, `rtf`, `txt`, `epub`, `md`, `html` = a zip), `spreadsheet` (`xlsx`, `ods`, `pdf`, `csv`, `tsv`), `presentation` (`pptx`, `odp`, `pdf`, `txt`), `drawing` (`png`, `svg`, `pdf`, `jpg`). |
|
|
53
|
+
| `origins` | `DEFAULT_GOOGLE_DRIVE_ORIGINS` | `drive`, `usercontent`, `docs`, `accounts`, `api` — tests point them at a stand-in. |
|
|
54
|
+
|
|
55
|
+
## Security and limits
|
|
56
|
+
|
|
57
|
+
- Every request is the reader's guarded fetch — its address guard, plugins, transport and timeout,
|
|
58
|
+
and the collect's abort signal; the source opens no connection of its own.
|
|
59
|
+
- Only public items are read. A private link fails with `llm-reader:fetch:google-drive:not-public`,
|
|
60
|
+
an exhausted quota with `quota`, a missing item with `not-found`, a refused key with `api-key` —
|
|
61
|
+
the collect's one failed item. A private file inside a public folder is left out and logged.
|
|
62
|
+
- Entry paths are relative to the linked folder; names lose `/`, `\` and control characters, and
|
|
63
|
+
the collector confines every path.
|
|
64
|
+
- The walk is bounded by `depth`, `maxEntries`, `maxFolders` and the collect's `maxFiles`; a cut is
|
|
65
|
+
logged, never thrown. A probe reads at most 256 KiB of an interstitial page.
|
|
66
|
+
- Logs carry ids, counts and reasons only — never an API key or a URL query.
|
|
67
|
+
|
|
68
|
+
## API
|
|
69
|
+
|
|
70
|
+
| Export | Purpose |
|
|
71
|
+
|---|---|
|
|
72
|
+
| `createGoogleDriveSource(options?)` | A `ReaderSource` named `google-drive`: `matches` (the URL alone), `recognizes` (a link that landed on Drive), `expand`. |
|
|
73
|
+
| `createGoogleDriveUrlHelper(options?)` | `parse(url)` → `GoogleDriveLink`, `downloadUrl`, `exportOf`, `folderViewUrl`, `apiListUrl`, `apiFileUrl`, `isSignIn`, `owns`. |
|
|
74
|
+
| `GoogleDriveError` | A `ReaderFetchError`: `llm-reader:fetch:google-drive:<reason>[:<detail>]`, `reason: GoogleDriveFailure`. |
|
|
75
|
+
| `GoogleDriveFailure` | `not-public`, `quota`, `not-found`, `api-key`, `not-a-link`, `unrecognized`, `resolver`, `status`. |
|
|
76
|
+
| `GoogleDriveKind` | `folder`, `file`, `document`, `spreadsheet`, `presentation`, `drawing`, `unresolved` (`open?id=`). |
|
|
77
|
+
| `GOOGLE_DRIVE_SOURCE`, `DEFAULT_GOOGLE_DRIVE_ORIGINS`, `DEFAULT_GOOGLE_EXPORTS`, `DEFAULT_GOOGLE_DRIVE_DEPTH`, `DEFAULT_GOOGLE_DRIVE_MAX_ENTRIES`, `DEFAULT_GOOGLE_DRIVE_MAX_FOLDERS` | `'google-drive'`, Google's hosts, the export defaults, 3, 200, 50. |
|
|
78
|
+
| `GoogleDriveSourceOptions`, `GoogleDriveUrlOptions`, `GoogleDriveOrigins`, `GoogleExportFormats`, `GoogleDriveLink`, `GoogleEditorKind`, `GoogleExport`, `GoogleDriveUrlHelper` | Types. |
|
|
79
|
+
|
|
80
|
+
The package has one entry point; there are no subpath exports.
|
|
81
|
+
|
|
82
|
+
## Common pitfalls
|
|
83
|
+
|
|
84
|
+
- Keyless folders come from Drive's static embedded folder view; files are probed with one ranged GET
|
|
85
|
+
for their name, size and — behind Drive's "can't scan for viruses" form — the URL that serves the
|
|
86
|
+
bytes. With `apiKey` sizes come from the API and only a file past Drive's 100 MiB scan limit is
|
|
87
|
+
probed.
|
|
88
|
+
- With file-streams' pinned fetch injected into the reader, build it with `resolvers: []`: its Drive
|
|
89
|
+
resolver refuses folder pages (`google-drive:resolver`) and rewrites exports to its own formats.
|
|
90
|
+
- An API key sees public items only; a private item answers 404 (`not-found`) through the API.
|
|
91
|
+
|
|
92
|
+
## Related packages
|
|
93
|
+
|
|
94
|
+
- [`@owlmeans/llm-reader`](https://github.com/owlmeans/common/tree/main/packages/llm-reader) — the reader, `ReaderSource`, the collector, sinks, guard and limits
|
|
95
|
+
- [`@owlmeans/file-streams`](https://github.com/owlmeans/common/tree/main/packages/file-streams) — `@owlmeans/file-streams/fetch`: the pinned `fetch` a reader can take
|
|
96
|
+
- [`@owlmeans/llm-reader-playwright`](https://github.com/owlmeans/common/tree/main/packages/llm-reader-playwright) — the reader's browser renderer
|
|
97
|
+
|
|
98
|
+
The `llm-reader-google-drive` skill covers the same source for agents.
|
|
99
|
+
|
|
100
|
+
<!-- owlmeans:agent-guidance:start -->
|
|
101
|
+
## Agent guidance
|
|
102
|
+
|
|
103
|
+
This package ships embedded agent skills under `agent-meta/`. After installing your
|
|
104
|
+
`@owlmeans/*` packages, run the OwlMeans agent-skills installer to place them into
|
|
105
|
+
your project's skill store (`.agents/skills/`):
|
|
106
|
+
|
|
107
|
+
```sh
|
|
108
|
+
npx @owlmeans/agent-skills@^0.2.0
|
|
109
|
+
```
|
|
110
|
+
|
|
111
|
+
The embedded files are version-matched to this package release. Do not edit them
|
|
112
|
+
directly — they are regenerated on each publish. To contribute guidance edits,
|
|
113
|
+
open a PR against the source monorepo.
|
|
114
|
+
<!-- owlmeans:agent-guidance:end -->
|
|
@@ -0,0 +1,16 @@
|
|
|
1
|
+
{
|
|
2
|
+
"schemaVersion": 3,
|
|
3
|
+
"package": "@owlmeans/llm-reader-google-drive",
|
|
4
|
+
"version": "0.2.0",
|
|
5
|
+
"generatedAt": "2026-11-15T13:38:38.781Z",
|
|
6
|
+
"canonicalRepo": "https://github.com/owlmeans/connect",
|
|
7
|
+
"entries": [
|
|
8
|
+
{
|
|
9
|
+
"kind": "skill",
|
|
10
|
+
"name": "llm-reader-google-drive",
|
|
11
|
+
"category": "package-specific",
|
|
12
|
+
"file": "skills/llm-reader-google-drive/SKILL.md",
|
|
13
|
+
"canonicalPath": ".agents/skills/llm-reader-google-drive/SKILL.md"
|
|
14
|
+
}
|
|
15
|
+
]
|
|
16
|
+
}
|
|
@@ -0,0 +1,131 @@
|
|
|
1
|
+
---
|
|
2
|
+
name: llm-reader-google-drive
|
|
3
|
+
description: "How to use @owlmeans/llm-reader-google-drive (OwlMeans connect) — the Google Drive ReaderSource of @owlmeans/llm-reader: createGoogleDriveSource (public folder links listed keyless from Drive's embedded folder view or through Drive API v3 with an API key, breadth-first sub-folders bounded by depth/maxEntries/maxFolders and the collect's maxFiles, file links turned into direct usercontent downloads probed for name, size and Drive's virus-scan form, Docs/Sheets/Slides/Drawings exported in configurable formats, open?id= and shortened links resolved), createGoogleDriveUrlHelper, GoogleDriveError reasons (not-public, quota, not-found, api-key, resolver) and the pinned-fetch caveat. Auto-invoked when collecting a Google Drive or Google Docs link with the reader, wiring reader sources, or debugging a Drive link that came back failed, empty or as HTML."
|
|
4
|
+
user-invocable: false
|
|
5
|
+
---
|
|
6
|
+
<!-- AUTO-GENERATED — do not edit. Regenerate via sync-agent-meta. -->
|
|
7
|
+
|
|
8
|
+
# @owlmeans/llm-reader-google-drive
|
|
9
|
+
|
|
10
|
+
**Layer:** Server (Node / Bun — wherever `@owlmeans/llm-reader` runs)
|
|
11
|
+
**Repo:** OwlMeans connect — the Drive half; the collector, sinks, guard and limits are common's
|
|
12
|
+
`@owlmeans/llm-reader` (its `collect` part)
|
|
13
|
+
**Install:** `"@owlmeans/llm-reader-google-drive": "^0.2.0"` beside `@owlmeans/llm-reader`
|
|
14
|
+
|
|
15
|
+
## Public API
|
|
16
|
+
|
|
17
|
+
| Symbol | Kind | Purpose |
|
|
18
|
+
|---|---|---|
|
|
19
|
+
| `createGoogleDriveSource(opts?)` | factory | A `ReaderSource` named `google-drive`: `matches` (URL alone), `recognizes` (a link that landed on Drive), `expand` |
|
|
20
|
+
| `createGoogleDriveUrlHelper(opts?)` | helper | `parse(url)` → `GoogleDriveLink` (`kind`, `id`, `resourceKey?`), `downloadUrl`, `exportOf`, `folderViewUrl`, `apiListUrl`, `apiFileUrl`, `isSignIn`, `owns` |
|
|
21
|
+
| `GoogleDriveSourceOptions` | interface | `apiKey?`, `depth?` (3), `maxEntries?` (200), `maxFolders?` (50), `exports?`, `origins?` |
|
|
22
|
+
| `GoogleExportFormats` | interface | `document` (`docx` · pdf, odt, rtf, txt, epub, md, html = zip), `spreadsheet` (`xlsx` · ods, pdf, csv, tsv), `presentation` (`pptx` · odp, pdf, txt), `drawing` (`png` · svg, pdf, jpg) |
|
|
23
|
+
| `GoogleDriveOrigins` | interface | `drive`, `usercontent`, `docs`, `accounts`, `api` — tests point them at a stand-in |
|
|
24
|
+
| `GoogleDriveError` | error | `ReaderFetchError` subclass, message `llm-reader:fetch:google-drive:<reason>`, `reason: GoogleDriveFailure` |
|
|
25
|
+
| `GoogleDriveFailure` | enum | `not-public`, `quota`, `not-found`, `api-key`, `not-a-link`, `unrecognized`, `resolver`, `status` |
|
|
26
|
+
| `GoogleDriveKind` | enum | `folder`, `file`, `document`, `spreadsheet`, `presentation`, `drawing`, `unresolved` (`open?id=`) |
|
|
27
|
+
| `GOOGLE_DRIVE_SOURCE`, `DEFAULT_GOOGLE_DRIVE_ORIGINS`, `DEFAULT_GOOGLE_EXPORTS`, `DEFAULT_GOOGLE_DRIVE_{DEPTH,MAX_ENTRIES,MAX_FOLDERS}` | const | `'google-drive'`, Google's hosts, the export defaults, 3 / 200 / 50 |
|
|
28
|
+
|
|
29
|
+
## Usage
|
|
30
|
+
|
|
31
|
+
```ts
|
|
32
|
+
import { createDirectoryCollectSink, createLlmReader } from '@owlmeans/llm-reader'
|
|
33
|
+
import { createGoogleDriveSource } from '@owlmeans/llm-reader-google-drive'
|
|
34
|
+
|
|
35
|
+
const reader = createLlmReader({ sources: [createGoogleDriveSource({ exports: { document: 'md' } })] })
|
|
36
|
+
const result = await reader.collect(link, { sink: createDirectoryCollectSink(dir), prefix: 'links/01-drive' })
|
|
37
|
+
```
|
|
38
|
+
|
|
39
|
+
There is no source host in a context: a consumer passes the source in `LlmReaderOptions.sources`
|
|
40
|
+
(also through `appendLlmReaderService(ctx, { sources })`). Sources are consulted in order, before the
|
|
41
|
+
reader's built-in S3 source.
|
|
42
|
+
|
|
43
|
+
## What expand does
|
|
44
|
+
|
|
45
|
+
- **Links** (`parse`): `drive.google.com/drive/[u/<n>/|mobile/]folders/<id>`, `[embedded]folderview?id=`,
|
|
46
|
+
`file/[u/<n>/]d/<id>`, `uc?id=`, `open?id=` (unresolved), `drive.usercontent.google.com/download?id=`,
|
|
47
|
+
`docs.google.com/{document,spreadsheets,presentation,drawings}/[u/<n>/]d/<id>`; http or https; ids
|
|
48
|
+
`[A-Za-z0-9_-]{10,}`; `resourcekey` kept. Forms, published `/d/e/…` documents and other hosts are not links.
|
|
49
|
+
- **`open?id=`**: keyless, Drive's 307 is followed and the landing URL parsed (fallback: a file); with a
|
|
50
|
+
key, `files.get` tells the type.
|
|
51
|
+
- **Folders**, breadth-first from level 0 (the linked folder) while `level < depth` for sub-folders;
|
|
52
|
+
entries stop at `min(maxEntries, limits.maxFiles + 1)` (one over, so the collector reports the
|
|
53
|
+
`max-files` cut) and listings at `maxFolders`; a cut is a `warn` line, never an error. A folder seen
|
|
54
|
+
twice is listed once. A sub-folder that fails is skipped (`warn`); the linked folder's failure throws.
|
|
55
|
+
- Keyless: `GET <drive>/embeddedfolderview?id=<id>[&resourcekey=]` — static HTML parsed with
|
|
56
|
+
linkedom: `div.flip-entry` (`id="entry-<id>"`), its `a[href]` (`/file/d/` file, `/drive/folders/`
|
|
57
|
+
sub-folder, `docs.google.com/…` editor file), `.flip-entry-title` (the name), the icon's
|
|
58
|
+
`…/type/<media type>` (the entry `mime`). No sizes. Other Google types (forms, sites) are left out.
|
|
59
|
+
- With `apiKey`: `GET <api>/drive/v3/files?q='<id>' in parents and trashed=false` (all drives, 1000
|
|
60
|
+
per page, ≤ 20 pages, paging stops once enough files are listed), header `X-Goog-Api-Key` (never
|
|
61
|
+
the URL) and `X-Goog-Drive-Resource-Keys` for keyed links; sizes and media types from the API; a
|
|
62
|
+
shortcut is listed as its target.
|
|
63
|
+
- **Entry paths** are `<sub-folder>/<name>` relative to the linked folder (the collector adds its
|
|
64
|
+
prefix and dedupes `-2`); a name loses `/`, `\` and control characters, `.`/`..` become `_`, the
|
|
65
|
+
extension stays.
|
|
66
|
+
- **Stored files** become `<usercontent>/download?id=<id>&export=download&confirm=t[&resourcekey=]`.
|
|
67
|
+
A file whose size is unknown (keyless) or at least 100 MiB (Drive's scan limit) and not over the
|
|
68
|
+
collect's `maxFileBytes` is **probed**: `GET` with `Range: bytes=0-0` (body ≤ 256 KiB) — an
|
|
69
|
+
attachment gives the name (`Content-Disposition`, `filename*` first) and size (`Content-Range`);
|
|
70
|
+
an HTML answer is Drive's warning page, whose `form#download-form` action and hidden inputs (or the
|
|
71
|
+
legacy `uc?export=download&confirm=` link, on Drive's own hosts only) become the entry URL, ≤ 3
|
|
72
|
+
steps. A probe Drive refuses leaves the file out (`warn`); a probe that fails on the wire keeps the
|
|
73
|
+
plain URL for the collector to report.
|
|
74
|
+
- **Editor files** become `docs.google.com/document/d/<id>/export?format=<f>`,
|
|
75
|
+
`/spreadsheets/d/<id>/export?format=<f>`, `/presentation/d/<id>/export/<f>`,
|
|
76
|
+
`/drawings/d/<id>/export/<f>`; the name takes the format's extension (`html` → `.zip`).
|
|
77
|
+
- **A single file or editor link** is one entry, probed for its name (the export's
|
|
78
|
+
`Content-Disposition` for an editor file).
|
|
79
|
+
|
|
80
|
+
## Rules
|
|
81
|
+
|
|
82
|
+
- Every request is `ReaderSourceContext.fetch` — the reader's guard, plugins, transport and timeout,
|
|
83
|
+
the collect's signal. Entry URLs are direct: a plain GET returns the bytes.
|
|
84
|
+
- Refusals throw `GoogleDriveError`: Google's sign-in host (any hop), 401/403 or a "you need access"
|
|
85
|
+
page → `not-public`; 429, a quota page or API rate-limit reasons → `quota`; 404 → `not-found` (an
|
|
86
|
+
API key also gets 404 for a private item); an invalid key or a disabled Drive API → `api-key`. The
|
|
87
|
+
collect turns it into the link's one `failed` item with the message as `reason`.
|
|
88
|
+
- **file-streams' pinned fetch**: inject it with `resolvers: []`. Its Drive resolver refuses the
|
|
89
|
+
folder page (`google-drive:resolver`) and rewrites exports to its own formats; single files work
|
|
90
|
+
through it either way.
|
|
91
|
+
- Logging scope `llm-reader-google-drive`: ids, counts and reasons only — never an API key or a URL query.
|
|
92
|
+
|
|
93
|
+
## Pitfalls
|
|
94
|
+
|
|
95
|
+
- A Drive link that comes back as one `failed` item names its reason: `not-public` (share the item
|
|
96
|
+
as "anyone with the link"), `quota` (retry later), `not-found`, `api-key`.
|
|
97
|
+
- Keyless listing has no sizes: every listed file costs one ranged probe request for its name and
|
|
98
|
+
size; with `apiKey` only files at or past Drive's 100 MiB scan limit are probed.
|
|
99
|
+
- A pinned fetch built without `resolvers: []` breaks folder links (`google-drive:resolver`).
|
|
100
|
+
|
|
101
|
+
## Tests
|
|
102
|
+
|
|
103
|
+
`bun test ./tests` in the package.
|
|
104
|
+
|
|
105
|
+
- `source.spec` runs a real reader over a real local stand-in of Google (`startDriveStandIn` in the
|
|
106
|
+
test context: Drive's embedded-view markup, `open?id=` redirects, ranged usercontent downloads, the
|
|
107
|
+
warning form, a quota page, Docs exports behind a redirect, the sign-in host, Drive API v3 in pages
|
|
108
|
+
of two with a refused key, a shortener) plus direct `expand` over plain `fetch` (`plainContext`):
|
|
109
|
+
link parsing and URL building, a folder collected with sub-folders and an exported Doc, the
|
|
110
|
+
depth/`maxEntries`/`maxFiles` cut, export options and single links, every refusal, the API-key path
|
|
111
|
+
with the key only as a header.
|
|
112
|
+
- `live.spec` is gated by `LLM_READER_GDRIVE_LIVE=1`: a shared folder of three files checked by
|
|
113
|
+
sha256, and one file link.
|
|
114
|
+
- When Drive's markup or redirects change, mirror Google's answer in the stand-in first, then
|
|
115
|
+
change the lister or probe; `origins` points every Google host at the stand-in.
|
|
116
|
+
|
|
117
|
+
## Related
|
|
118
|
+
|
|
119
|
+
- [[llm-reader]] (`@owlmeans/llm-reader`) — the reader, `ReaderSource`, the collector, sinks, guard and limits (its `collect` part)
|
|
120
|
+
- [[file-streams]] (`@owlmeans/file-streams`) — the pinned `fetch` (`@owlmeans/file-streams/fetch`) to build with `resolvers: []`
|
|
121
|
+
|
|
122
|
+
## External docs
|
|
123
|
+
|
|
124
|
+
- https://developers.google.com/drive/api/reference/rest/v3/files/list — `q`, `fields`, `pageToken`,
|
|
125
|
+
`supportsAllDrives`/`includeItemsFromAllDrives`; API keys reach public files only.
|
|
126
|
+
- https://developers.google.com/drive/api/guides/resource-keys — `X-Goog-Drive-Resource-Keys: <id>/<key>`
|
|
127
|
+
for links shared before the 2021 security update.
|
|
128
|
+
- https://cloud.google.com/docs/authentication/api-keys-use — `X-Goog-Api-Key` header instead of `key=`.
|
|
129
|
+
- Observed 2026-11: `embeddedfolderview` answers static HTML without cookies; usercontent honours
|
|
130
|
+
`Range` (206 with `Content-Range`) and serves `application/octet-stream` with an attachment
|
|
131
|
+
disposition; `open?id=` is a 307 to the item's page; `uc?id=` a 303 to usercontent; a missing item is 404.
|
|
@@ -0,0 +1,45 @@
|
|
|
1
|
+
import type { GoogleDriveOrigins, GoogleExportFormats } from './types.js';
|
|
2
|
+
/** The name the source expands links under (`CollectResult.source`). */
|
|
3
|
+
export declare const GOOGLE_DRIVE_SOURCE = "google-drive";
|
|
4
|
+
/** Where Google answers; tests point these at a stand-in. */
|
|
5
|
+
export declare const DEFAULT_GOOGLE_DRIVE_ORIGINS: Readonly<Required<GoogleDriveOrigins>>;
|
|
6
|
+
/** The format each Google editor file is exported as unless `exports` names another. */
|
|
7
|
+
export declare const DEFAULT_GOOGLE_EXPORTS: Readonly<Required<GoogleExportFormats>>;
|
|
8
|
+
/** Sub-folder levels listed below the linked folder (`0` = its own files only). */
|
|
9
|
+
export declare const DEFAULT_GOOGLE_DRIVE_DEPTH = 3;
|
|
10
|
+
/** Entries one link expands to at most — under the collect's own `maxFiles + 1`. */
|
|
11
|
+
export declare const DEFAULT_GOOGLE_DRIVE_MAX_ENTRIES = 200;
|
|
12
|
+
/** Folders one link lists at most (each is one request). */
|
|
13
|
+
export declare const DEFAULT_GOOGLE_DRIVE_MAX_FOLDERS = 50;
|
|
14
|
+
/** What a Drive link points at. */
|
|
15
|
+
export declare enum GoogleDriveKind {
|
|
16
|
+
Folder = "folder",
|
|
17
|
+
/** A stored file — bytes behind the usercontent download. */
|
|
18
|
+
File = "file",
|
|
19
|
+
Document = "document",
|
|
20
|
+
Spreadsheet = "spreadsheet",
|
|
21
|
+
Presentation = "presentation",
|
|
22
|
+
Drawing = "drawing",
|
|
23
|
+
/** An `open?id=` link: Drive itself tells what it is. */
|
|
24
|
+
Unresolved = "unresolved"
|
|
25
|
+
}
|
|
26
|
+
/** Why a link or an entry could not be expanded — the `reason` of a `GoogleDriveError`. */
|
|
27
|
+
export declare enum GoogleDriveFailure {
|
|
28
|
+
/** Shared privately: Google answered with its sign-in page, 401/403 or "you need access". */
|
|
29
|
+
NotPublic = "not-public",
|
|
30
|
+
/** Download or API quota exhausted ("too many users have viewed or downloaded", 429, rate limits). */
|
|
31
|
+
Quota = "quota",
|
|
32
|
+
/** Drive answered 404 — no such item, or one an API key may not see. */
|
|
33
|
+
NotFound = "not-found",
|
|
34
|
+
/** The Drive API refused the key (invalid, or the Drive API not enabled for it). */
|
|
35
|
+
ApiKey = "api-key",
|
|
36
|
+
/** The URL is not a Drive or Docs link this source reads. */
|
|
37
|
+
NotALink = "not-a-link",
|
|
38
|
+
/** A Drive answer this source does not know how to read. */
|
|
39
|
+
Unrecognized = "unrecognized",
|
|
40
|
+
/** A file-streams Drive resolver inside the injected fetch refused the folder page. */
|
|
41
|
+
Resolver = "resolver",
|
|
42
|
+
/** Any other refusing status (`status:<code>`). */
|
|
43
|
+
Status = "status"
|
|
44
|
+
}
|
|
45
|
+
//# sourceMappingURL=consts.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"consts.d.ts","sourceRoot":"","sources":["../src/consts.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,kBAAkB,EAAE,mBAAmB,EAAE,MAAM,YAAY,CAAA;AAEzE,wEAAwE;AACxE,eAAO,MAAM,mBAAmB,iBAAiB,CAAA;AAEjD,6DAA6D;AAC7D,eAAO,MAAM,4BAA4B,EAAE,QAAQ,CAAC,QAAQ,CAAC,kBAAkB,CAAC,CAM9E,CAAA;AAEF,wFAAwF;AACxF,eAAO,MAAM,sBAAsB,EAAE,QAAQ,CAAC,QAAQ,CAAC,mBAAmB,CAAC,CAEzE,CAAA;AAEF,mFAAmF;AACnF,eAAO,MAAM,0BAA0B,IAAI,CAAA;AAE3C,oFAAoF;AACpF,eAAO,MAAM,gCAAgC,MAAM,CAAA;AAEnD,4DAA4D;AAC5D,eAAO,MAAM,gCAAgC,KAAK,CAAA;AAElD,mCAAmC;AACnC,oBAAY,eAAe;IACzB,MAAM,WAAW;IACjB,6DAA6D;IAC7D,IAAI,SAAS;IACb,QAAQ,aAAa;IACrB,WAAW,gBAAgB;IAC3B,YAAY,iBAAiB;IAC7B,OAAO,YAAY;IACnB,yDAAyD;IACzD,UAAU,eAAe;CAC1B;AAED,2FAA2F;AAC3F,oBAAY,kBAAkB;IAC5B,6FAA6F;IAC7F,SAAS,eAAe;IACxB,sGAAsG;IACtG,KAAK,UAAU;IACf,wEAAwE;IACxE,QAAQ,cAAc;IACtB,oFAAoF;IACpF,MAAM,YAAY;IAClB,6DAA6D;IAC7D,QAAQ,eAAe;IACvB,4DAA4D;IAC5D,YAAY,iBAAiB;IAC7B,uFAAuF;IACvF,QAAQ,aAAa;IACrB,mDAAmD;IACnD,MAAM,WAAW;CAClB"}
|
package/build/consts.js
ADDED
|
@@ -0,0 +1,54 @@
|
|
|
1
|
+
/** The name the source expands links under (`CollectResult.source`). */
|
|
2
|
+
export const GOOGLE_DRIVE_SOURCE = 'google-drive';
|
|
3
|
+
/** Where Google answers; tests point these at a stand-in. */
|
|
4
|
+
export const DEFAULT_GOOGLE_DRIVE_ORIGINS = Object.freeze({
|
|
5
|
+
drive: 'https://drive.google.com',
|
|
6
|
+
usercontent: 'https://drive.usercontent.google.com',
|
|
7
|
+
docs: 'https://docs.google.com',
|
|
8
|
+
accounts: 'https://accounts.google.com',
|
|
9
|
+
api: 'https://www.googleapis.com',
|
|
10
|
+
});
|
|
11
|
+
/** The format each Google editor file is exported as unless `exports` names another. */
|
|
12
|
+
export const DEFAULT_GOOGLE_EXPORTS = Object.freeze({
|
|
13
|
+
document: 'docx', spreadsheet: 'xlsx', presentation: 'pptx', drawing: 'png',
|
|
14
|
+
});
|
|
15
|
+
/** Sub-folder levels listed below the linked folder (`0` = its own files only). */
|
|
16
|
+
export const DEFAULT_GOOGLE_DRIVE_DEPTH = 3;
|
|
17
|
+
/** Entries one link expands to at most — under the collect's own `maxFiles + 1`. */
|
|
18
|
+
export const DEFAULT_GOOGLE_DRIVE_MAX_ENTRIES = 200;
|
|
19
|
+
/** Folders one link lists at most (each is one request). */
|
|
20
|
+
export const DEFAULT_GOOGLE_DRIVE_MAX_FOLDERS = 50;
|
|
21
|
+
/** What a Drive link points at. */
|
|
22
|
+
export var GoogleDriveKind;
|
|
23
|
+
(function (GoogleDriveKind) {
|
|
24
|
+
GoogleDriveKind["Folder"] = "folder";
|
|
25
|
+
/** A stored file — bytes behind the usercontent download. */
|
|
26
|
+
GoogleDriveKind["File"] = "file";
|
|
27
|
+
GoogleDriveKind["Document"] = "document";
|
|
28
|
+
GoogleDriveKind["Spreadsheet"] = "spreadsheet";
|
|
29
|
+
GoogleDriveKind["Presentation"] = "presentation";
|
|
30
|
+
GoogleDriveKind["Drawing"] = "drawing";
|
|
31
|
+
/** An `open?id=` link: Drive itself tells what it is. */
|
|
32
|
+
GoogleDriveKind["Unresolved"] = "unresolved";
|
|
33
|
+
})(GoogleDriveKind || (GoogleDriveKind = {}));
|
|
34
|
+
/** Why a link or an entry could not be expanded — the `reason` of a `GoogleDriveError`. */
|
|
35
|
+
export var GoogleDriveFailure;
|
|
36
|
+
(function (GoogleDriveFailure) {
|
|
37
|
+
/** Shared privately: Google answered with its sign-in page, 401/403 or "you need access". */
|
|
38
|
+
GoogleDriveFailure["NotPublic"] = "not-public";
|
|
39
|
+
/** Download or API quota exhausted ("too many users have viewed or downloaded", 429, rate limits). */
|
|
40
|
+
GoogleDriveFailure["Quota"] = "quota";
|
|
41
|
+
/** Drive answered 404 — no such item, or one an API key may not see. */
|
|
42
|
+
GoogleDriveFailure["NotFound"] = "not-found";
|
|
43
|
+
/** The Drive API refused the key (invalid, or the Drive API not enabled for it). */
|
|
44
|
+
GoogleDriveFailure["ApiKey"] = "api-key";
|
|
45
|
+
/** The URL is not a Drive or Docs link this source reads. */
|
|
46
|
+
GoogleDriveFailure["NotALink"] = "not-a-link";
|
|
47
|
+
/** A Drive answer this source does not know how to read. */
|
|
48
|
+
GoogleDriveFailure["Unrecognized"] = "unrecognized";
|
|
49
|
+
/** A file-streams Drive resolver inside the injected fetch refused the folder page. */
|
|
50
|
+
GoogleDriveFailure["Resolver"] = "resolver";
|
|
51
|
+
/** Any other refusing status (`status:<code>`). */
|
|
52
|
+
GoogleDriveFailure["Status"] = "status";
|
|
53
|
+
})(GoogleDriveFailure || (GoogleDriveFailure = {}));
|
|
54
|
+
//# sourceMappingURL=consts.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"consts.js","sourceRoot":"","sources":["../src/consts.ts"],"names":[],"mappings":"AAEA,wEAAwE;AACxE,MAAM,CAAC,MAAM,mBAAmB,GAAG,cAAc,CAAA;AAEjD,6DAA6D;AAC7D,MAAM,CAAC,MAAM,4BAA4B,GAA2C,MAAM,CAAC,MAAM,CAAC;IAChG,KAAK,EAAE,0BAA0B;IACjC,WAAW,EAAE,sCAAsC;IACnD,IAAI,EAAE,yBAAyB;IAC/B,QAAQ,EAAE,6BAA6B;IACvC,GAAG,EAAE,4BAA4B;CAClC,CAAC,CAAA;AAEF,wFAAwF;AACxF,MAAM,CAAC,MAAM,sBAAsB,GAA4C,MAAM,CAAC,MAAM,CAAC;IAC3F,QAAQ,EAAE,MAAM,EAAE,WAAW,EAAE,MAAM,EAAE,YAAY,EAAE,MAAM,EAAE,OAAO,EAAE,KAAK;CAC5E,CAAC,CAAA;AAEF,mFAAmF;AACnF,MAAM,CAAC,MAAM,0BAA0B,GAAG,CAAC,CAAA;AAE3C,oFAAoF;AACpF,MAAM,CAAC,MAAM,gCAAgC,GAAG,GAAG,CAAA;AAEnD,4DAA4D;AAC5D,MAAM,CAAC,MAAM,gCAAgC,GAAG,EAAE,CAAA;AAElD,mCAAmC;AACnC,MAAM,CAAN,IAAY,eAUX;AAVD,WAAY,eAAe;IACzB,oCAAiB,CAAA;IACjB,6DAA6D;IAC7D,gCAAa,CAAA;IACb,wCAAqB,CAAA;IACrB,8CAA2B,CAAA;IAC3B,gDAA6B,CAAA;IAC7B,sCAAmB,CAAA;IACnB,yDAAyD;IACzD,4CAAyB,CAAA;AAC3B,CAAC,EAVW,eAAe,KAAf,eAAe,QAU1B;AAED,2FAA2F;AAC3F,MAAM,CAAN,IAAY,kBAiBX;AAjBD,WAAY,kBAAkB;IAC5B,6FAA6F;IAC7F,8CAAwB,CAAA;IACxB,sGAAsG;IACtG,qCAAe,CAAA;IACf,wEAAwE;IACxE,4CAAsB,CAAA;IACtB,oFAAoF;IACpF,wCAAkB,CAAA;IAClB,6DAA6D;IAC7D,6CAAuB,CAAA;IACvB,4DAA4D;IAC5D,mDAA6B,CAAA;IAC7B,uFAAuF;IACvF,2CAAqB,CAAA;IACrB,mDAAmD;IACnD,uCAAiB,CAAA;AACnB,CAAC,EAjBW,kBAAkB,KAAlB,kBAAkB,QAiB7B"}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"consts.local.d.ts","sourceRoot":"","sources":["../src/consts.local.ts"],"names":[],"mappings":"AAAA,gGAAgG;AAChG,eAAO,MAAM,gBAAgB,QAAyB,CAAA"}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"consts.local.js","sourceRoot":"","sources":["../src/consts.local.ts"],"names":[],"mappings":"AAAA,gGAAgG;AAChG,MAAM,CAAC,MAAM,gBAAgB,GAAG,sBAAsB,CAAA"}
|
|
@@ -0,0 +1,14 @@
|
|
|
1
|
+
import { ReaderFetchError } from '@owlmeans/llm-reader';
|
|
2
|
+
import { GoogleDriveFailure } from './consts.js';
|
|
3
|
+
/**
|
|
4
|
+
* A Drive link or entry that could not be expanded — `llm-reader:fetch:google-drive:<reason>[:<detail>]`.
|
|
5
|
+
* A reader fetch error, so a collect reports it as the link's failed item with this message as the reason.
|
|
6
|
+
*/
|
|
7
|
+
export declare class GoogleDriveError extends ReaderFetchError {
|
|
8
|
+
static typeName: string;
|
|
9
|
+
/** Why, read back from the message. */
|
|
10
|
+
reason: GoogleDriveFailure;
|
|
11
|
+
constructor(message?: string);
|
|
12
|
+
finalizeUnmarshal(): void;
|
|
13
|
+
}
|
|
14
|
+
//# sourceMappingURL=errors.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"errors.d.ts","sourceRoot":"","sources":["../src/errors.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,gBAAgB,EAAE,MAAM,sBAAsB,CAAA;AACvD,OAAO,EAAuB,kBAAkB,EAAE,MAAM,aAAa,CAAA;AAErE;;;GAGG;AACH,qBAAa,gBAAiB,SAAQ,gBAAgB;IACpD,OAAuB,QAAQ,SAA4C;IAE3E,uCAAuC;IAChC,MAAM,EAAE,kBAAkB,CAAkC;IAEnE,YAAY,OAAO,GAAE,MAAwC,EAI5D;IAEe,iBAAiB,IAAI,IAAI,CAMxC;CACF"}
|
package/build/errors.js
ADDED
|
@@ -0,0 +1,26 @@
|
|
|
1
|
+
import { ResilientError } from '@owlmeans/error';
|
|
2
|
+
import { ReaderFetchError } from '@owlmeans/llm-reader';
|
|
3
|
+
import { GOOGLE_DRIVE_SOURCE, GoogleDriveFailure } from './consts.js';
|
|
4
|
+
/**
|
|
5
|
+
* A Drive link or entry that could not be expanded — `llm-reader:fetch:google-drive:<reason>[:<detail>]`.
|
|
6
|
+
* A reader fetch error, so a collect reports it as the link's failed item with this message as the reason.
|
|
7
|
+
*/
|
|
8
|
+
export class GoogleDriveError extends ReaderFetchError {
|
|
9
|
+
static typeName = `GoogleDrive${ReaderFetchError.typeName}`;
|
|
10
|
+
/** Why, read back from the message. */
|
|
11
|
+
reason = GoogleDriveFailure.Unrecognized;
|
|
12
|
+
constructor(message = GoogleDriveFailure.Unrecognized) {
|
|
13
|
+
super(`${GOOGLE_DRIVE_SOURCE}:${message}`);
|
|
14
|
+
this.type = GoogleDriveError.typeName;
|
|
15
|
+
this.finalizeUnmarshal();
|
|
16
|
+
}
|
|
17
|
+
finalizeUnmarshal() {
|
|
18
|
+
super.finalizeUnmarshal();
|
|
19
|
+
const reason = new RegExp(`${GOOGLE_DRIVE_SOURCE}:([a-z-]+)`).exec(this.message)?.[1];
|
|
20
|
+
this.reason = Object.values(GoogleDriveFailure).includes(reason)
|
|
21
|
+
? reason
|
|
22
|
+
: GoogleDriveFailure.Unrecognized;
|
|
23
|
+
}
|
|
24
|
+
}
|
|
25
|
+
ResilientError.registerErrorClass(GoogleDriveError);
|
|
26
|
+
//# sourceMappingURL=errors.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"errors.js","sourceRoot":"","sources":["../src/errors.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,cAAc,EAAE,MAAM,iBAAiB,CAAA;AAChD,OAAO,EAAE,gBAAgB,EAAE,MAAM,sBAAsB,CAAA;AACvD,OAAO,EAAE,mBAAmB,EAAE,kBAAkB,EAAE,MAAM,aAAa,CAAA;AAErE;;;GAGG;AACH,MAAM,OAAO,gBAAiB,SAAQ,gBAAgB;IAC7C,MAAM,CAAU,QAAQ,GAAG,cAAc,gBAAgB,CAAC,QAAQ,EAAE,CAAA;IAE3E,uCAAuC;IAChC,MAAM,GAAuB,kBAAkB,CAAC,YAAY,CAAA;IAEnE,YAAY,OAAO,GAAW,kBAAkB,CAAC,YAAY;QAC3D,KAAK,CAAC,GAAG,mBAAmB,IAAI,OAAO,EAAE,CAAC,CAAA;QAC1C,IAAI,CAAC,IAAI,GAAG,gBAAgB,CAAC,QAAQ,CAAA;QACrC,IAAI,CAAC,iBAAiB,EAAE,CAAA;IAC1B,CAAC;IAEe,iBAAiB;QAC/B,KAAK,CAAC,iBAAiB,EAAE,CAAA;QACzB,MAAM,MAAM,GAAG,IAAI,MAAM,CAAC,GAAG,mBAAmB,YAAY,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,EAAE,CAAC,CAAC,CAAC,CAAA;QACrF,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC,MAAM,CAAC,kBAAkB,CAAC,CAAC,QAAQ,CAAC,MAA4B,CAAC;YACpF,CAAC,CAAC,MAA4B;YAC9B,CAAC,CAAC,kBAAkB,CAAC,YAAY,CAAA;IACrC,CAAC;CACF;AAED,cAAc,CAAC,kBAAkB,CAAC,gBAAgB,CAAC,CAAA"}
|
package/build/index.d.ts
ADDED
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,mBAAmB,YAAY,CAAA;AAE/B,cAAc,aAAa,CAAA;AAC3B,cAAc,aAAa,CAAA;AAE3B,cAAc,UAAU,CAAA;AACxB,cAAc,aAAa,CAAA"}
|
package/build/index.js
ADDED
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAEA,cAAc,aAAa,CAAA;AAC3B,cAAc,aAAa,CAAA;AAE3B,cAAc,UAAU,CAAA;AACxB,cAAc,aAAa,CAAA"}
|
|
@@ -0,0 +1,9 @@
|
|
|
1
|
+
import type { GoogleDriveUrlHelper } from '../types.js';
|
|
2
|
+
import type { DriveHttp, DriveLister } from './types.local.js';
|
|
3
|
+
/**
|
|
4
|
+
* Folders listed through Drive API v3 with an API key (the `X-Goog-Api-Key` header, never the URL):
|
|
5
|
+
* `files.list` pages of `'<id>' in parents and trashed=false`, with sizes and media types; a
|
|
6
|
+
* shortcut is listed as its target; Google item types other than folders and editor files are left out.
|
|
7
|
+
*/
|
|
8
|
+
export declare const makeApiLister: (http: DriveHttp, urls: GoogleDriveUrlHelper, apiKey: string) => DriveLister;
|
|
9
|
+
//# sourceMappingURL=api.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"api.d.ts","sourceRoot":"","sources":["../../src/source/api.ts"],"names":[],"mappings":"AAIA,OAAO,KAAK,EAAE,oBAAoB,EAAE,MAAM,aAAa,CAAA;AAEvD,OAAO,KAAK,EAA8B,SAAS,EAAa,WAAW,EAAE,MAAM,kBAAkB,CAAA;AAIrG;;;;GAIG;AACH,eAAO,MAAM,aAAa,SAAU,SAAS,QAAQ,oBAAoB,UAAU,MAAM,KAAG,WAiE3F,CAAA"}
|
|
@@ -0,0 +1,80 @@
|
|
|
1
|
+
import { logger } from '@owlmeans/log';
|
|
2
|
+
import { GoogleDriveFailure, GoogleDriveKind } from '../consts.js';
|
|
3
|
+
import { DRIVE_ID_PATTERN } from '../consts.local.js';
|
|
4
|
+
import { GoogleDriveError } from '../errors.js';
|
|
5
|
+
import { DRIVE_API_MAX_PAGES, DRIVE_LISTING_BYTES, GOOGLE_APPS_KINDS, GOOGLE_APPS_PREFIX, GOOGLE_APPS_SHORTCUT, LOG_SCOPE } from './consts.local.js';
|
|
6
|
+
const log = logger(LOG_SCOPE);
|
|
7
|
+
/**
|
|
8
|
+
* Folders listed through Drive API v3 with an API key (the `X-Goog-Api-Key` header, never the URL):
|
|
9
|
+
* `files.list` pages of `'<id>' in parents and trashed=false`, with sizes and media types; a
|
|
10
|
+
* shortcut is listed as its target; Google item types other than folders and editor files are left out.
|
|
11
|
+
*/
|
|
12
|
+
export const makeApiLister = (http, urls, apiKey) => {
|
|
13
|
+
const headersOf = (id, resourceKey) => ({
|
|
14
|
+
'x-goog-api-key': apiKey,
|
|
15
|
+
accept: 'application/json',
|
|
16
|
+
...(resourceKey != null ? { 'x-goog-drive-resource-keys': `${id}/${resourceKey}` } : {}),
|
|
17
|
+
});
|
|
18
|
+
const json = async (url, headers) => {
|
|
19
|
+
const response = await http.get(url, { headers, maxBytes: DRIVE_LISTING_BYTES });
|
|
20
|
+
const refused = await http.refusal(response);
|
|
21
|
+
if (refused != null)
|
|
22
|
+
throw refused;
|
|
23
|
+
try {
|
|
24
|
+
return JSON.parse(await response.text());
|
|
25
|
+
}
|
|
26
|
+
catch {
|
|
27
|
+
throw new GoogleDriveError(`${GoogleDriveFailure.Unrecognized}:api-answer`);
|
|
28
|
+
}
|
|
29
|
+
};
|
|
30
|
+
const itemOf = (file) => {
|
|
31
|
+
const shortcut = file.mimeType === GOOGLE_APPS_SHORTCUT ? file.shortcutDetails : undefined;
|
|
32
|
+
const id = shortcut != null ? shortcut.targetId : file.id;
|
|
33
|
+
const mime = shortcut != null ? shortcut.targetMimeType : file.mimeType;
|
|
34
|
+
const resourceKey = shortcut != null ? shortcut.targetResourceKey : file.resourceKey;
|
|
35
|
+
if (id == null || !DRIVE_ID_PATTERN.test(id))
|
|
36
|
+
return undefined;
|
|
37
|
+
const native = mime != null && mime.startsWith(GOOGLE_APPS_PREFIX);
|
|
38
|
+
const kind = native ? GOOGLE_APPS_KINDS[mime] : GoogleDriveKind.File;
|
|
39
|
+
if (kind == null)
|
|
40
|
+
return undefined;
|
|
41
|
+
const size = Number(file.size ?? NaN);
|
|
42
|
+
return {
|
|
43
|
+
id, kind, name: file.name != null && file.name !== '' ? file.name : id,
|
|
44
|
+
...(mime != null && !native ? { mime } : {}), ...(Number.isFinite(size) ? { size } : {}),
|
|
45
|
+
...(resourceKey != null && resourceKey !== '' ? { resourceKey } : {}),
|
|
46
|
+
};
|
|
47
|
+
};
|
|
48
|
+
const list = async (folder, limit) => {
|
|
49
|
+
const items = [];
|
|
50
|
+
let token;
|
|
51
|
+
let unsupported = 0;
|
|
52
|
+
let files = 0;
|
|
53
|
+
for (let page = 0; page < DRIVE_API_MAX_PAGES; page++) {
|
|
54
|
+
const answer = await json(urls.apiListUrl(folder.id, token), headersOf(folder.id, folder.resourceKey));
|
|
55
|
+
for (const file of answer.files ?? []) {
|
|
56
|
+
const item = itemOf(file);
|
|
57
|
+
if (item == null) {
|
|
58
|
+
unsupported++;
|
|
59
|
+
continue;
|
|
60
|
+
}
|
|
61
|
+
items.push(item);
|
|
62
|
+
if (item.kind !== GoogleDriveKind.Folder)
|
|
63
|
+
files++;
|
|
64
|
+
}
|
|
65
|
+
token = answer.nextPageToken;
|
|
66
|
+
if (token == null || token === '' || files >= limit)
|
|
67
|
+
break;
|
|
68
|
+
}
|
|
69
|
+
log.debug('Listed a folder through the Drive API', { folder: folder.id, items: items.length, unsupported, more: token != null });
|
|
70
|
+
return items;
|
|
71
|
+
};
|
|
72
|
+
const resolve = async (_url, link) => {
|
|
73
|
+
const item = itemOf(await json(urls.apiFileUrl(link.id), headersOf(link.id, link.resourceKey)));
|
|
74
|
+
if (item == null)
|
|
75
|
+
throw new GoogleDriveError(`${GoogleDriveFailure.Unrecognized}:item-type`);
|
|
76
|
+
return { ...item };
|
|
77
|
+
};
|
|
78
|
+
return { list, resolve };
|
|
79
|
+
};
|
|
80
|
+
//# sourceMappingURL=api.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"api.js","sourceRoot":"","sources":["../../src/source/api.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,MAAM,EAAE,MAAM,eAAe,CAAA;AACtC,OAAO,EAAE,kBAAkB,EAAE,eAAe,EAAE,MAAM,cAAc,CAAA;AAClE,OAAO,EAAE,gBAAgB,EAAE,MAAM,oBAAoB,CAAA;AACrD,OAAO,EAAE,gBAAgB,EAAE,MAAM,cAAc,CAAA;AAE/C,OAAO,EAAE,mBAAmB,EAAE,mBAAmB,EAAE,iBAAiB,EAAE,kBAAkB,EAAE,oBAAoB,EAAE,SAAS,EAAE,MAAM,mBAAmB,CAAA;AAGpJ,MAAM,GAAG,GAAG,MAAM,CAAC,SAAS,CAAC,CAAA;AAE7B;;;;GAIG;AACH,MAAM,CAAC,MAAM,aAAa,GAAG,CAAC,IAAe,EAAE,IAA0B,EAAE,MAAc,EAAe,EAAE;IACxG,MAAM,SAAS,GAAG,CAAC,EAAU,EAAE,WAAoB,EAA0B,EAAE,CAAC,CAAC;QAC/E,gBAAgB,EAAE,MAAM;QACxB,MAAM,EAAE,kBAAkB;QAC1B,GAAG,CAAC,WAAW,IAAI,IAAI,CAAC,CAAC,CAAC,EAAE,4BAA4B,EAAE,GAAG,EAAE,IAAI,WAAW,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;KACzF,CAAC,CAAA;IAEF,MAAM,IAAI,GAAG,KAAK,EAAK,GAAW,EAAE,OAA+B,EAAc,EAAE;QACjF,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,GAAG,CAAC,GAAG,EAAE,EAAE,OAAO,EAAE,QAAQ,EAAE,mBAAmB,EAAE,CAAC,CAAA;QAChF,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAA;QAC5C,IAAI,OAAO,IAAI,IAAI;YAAE,MAAM,OAAO,CAAA;QAClC,IAAI,CAAC;YACH,OAAO,IAAI,CAAC,KAAK,CAAC,MAAM,QAAQ,CAAC,IAAI,EAAE,CAAM,CAAA;QAC/C,CAAC;QAAC,MAAM,CAAC;YACP,MAAM,IAAI,gBAAgB,CAAC,GAAG,kBAAkB,CAAC,YAAY,aAAa,CAAC,CAAA;QAC7E,CAAC;IACH,CAAC,CAAA;IAED,MAAM,MAAM,GAAG,CAAC,IAAkB,EAAyB,EAAE;QAC3D,MAAM,QAAQ,GAAG,IAAI,CAAC,QAAQ,KAAK,oBAAoB,CAAC,CAAC,CAAC,IAAI,CAAC,eAAe,CAAC,CAAC,CAAC,SAAS,CAAA;QAC1F,MAAM,EAAE,GAAG,QAAQ,IAAI,IAAI,CAAC,CAAC,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC,CAAC,IAAI,CAAC,EAAE,CAAA;QACzD,MAAM,IAAI,GAAG,QAAQ,IAAI,IAAI,CAAC,CAAC,CAAC,QAAQ,CAAC,cAAc,CAAC,CAAC,CAAC,IAAI,CAAC,QAAQ,CAAA;QACvE,MAAM,WAAW,GAAG,QAAQ,IAAI,IAAI,CAAC,CAAC,CAAC,QAAQ,CAAC,iBAAiB,CAAC,CAAC,CAAC,IAAI,CAAC,WAAW,CAAA;QACpF,IAAI,EAAE,IAAI,IAAI,IAAI,CAAC,gBAAgB,CAAC,IAAI,CAAC,EAAE,CAAC;YAAE,OAAO,SAAS,CAAA;QAC9D,MAAM,MAAM,GAAG,IAAI,IAAI,IAAI,IAAI,IAAI,CAAC,UAAU,CAAC,kBAAkB,CAAC,CAAA;QAClE,MAAM,IAAI,GAAG,MAAM,CAAC,CAAC,CAAC,iBAAiB,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,eAAe,CAAC,IAAI,CAAA;QACpE,IAAI,IAAI,IAAI,IAAI;YAAE,OAAO,SAAS,CAAA;QAClC,MAAM,IAAI,GAAG,MAAM,CAAC,IAAI,CAAC,IAAI,IAAI,GAAG,CAAC,CAAA;QACrC,OAAO;YACL,EAAE,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI,CAAC,IAAI,IAAI,IAAI,IAAI,IAAI,CAAC,IAAI,KAAK,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE;YACtE,GAAG,CAAC,IAAI,IAAI,IAAI,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,GAAG,CAAC,MAAM,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;YACxF,GAAG,CAAC,WAAW,IAAI,IAAI,IAAI,WAAW,KAAK,EAAE,CAAC,CAAC,CAAC,EAAE,WAAW,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;SACtE,CAAA;IACH,CAAC,CAAA;IAED,MAAM,IAAI,GAAwB,KAAK,EAAE,MAAM,EAAE,KAAK,EAAE,EAAE;QACxD,MAAM,KAAK,GAAgB,EAAE,CAAA;QAC7B,IAAI,KAAyB,CAAA;QAC7B,IAAI,WAAW,GAAG,CAAC,CAAA;QACnB,IAAI,KAAK,GAAG,CAAC,CAAA;QACb,KAAK,IAAI,IAAI,GAAG,CAAC,EAAE,IAAI,GAAG,mBAAmB,EAAE,IAAI,EAAE,EAAE,CAAC;YACtD,MAAM,MAAM,GAAG,MAAM,IAAI,CAAe,IAAI,CAAC,UAAU,CAAC,MAAM,CAAC,EAAE,EAAE,KAAK,CAAC,EAAE,SAAS,CAAC,MAAM,CAAC,EAAE,EAAE,MAAM,CAAC,WAAW,CAAC,CAAC,CAAA;YACpH,KAAK,MAAM,IAAI,IAAI,MAAM,CAAC,KAAK,IAAI,EAAE,EAAE,CAAC;gBACtC,MAAM,IAAI,GAAG,MAAM,CAAC,IAAI,CAAC,CAAA;gBACzB,IAAI,IAAI,IAAI,IAAI,EAAE,CAAC;oBACjB,WAAW,EAAE,CAAA;oBACb,SAAQ;gBACV,CAAC;gBACD,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;gBAChB,IAAI,IAAI,CAAC,IAAI,KAAK,eAAe,CAAC,MAAM;oBAAE,KAAK,EAAE,CAAA;YACnD,CAAC;YACD,KAAK,GAAG,MAAM,CAAC,aAAa,CAAA;YAC5B,IAAI,KAAK,IAAI,IAAI,IAAI,KAAK,KAAK,EAAE,IAAI,KAAK,IAAI,KAAK;gBAAE,MAAK;QAC5D,CAAC;QACD,GAAG,CAAC,KAAK,CAAC,uCAAuC,EAAE,EAAE,MAAM,EAAE,MAAM,CAAC,EAAE,EAAE,KAAK,EAAE,KAAK,CAAC,MAAM,EAAE,WAAW,EAAE,IAAI,EAAE,KAAK,IAAI,IAAI,EAAE,CAAC,CAAA;QAChI,OAAO,KAAK,CAAA;IACd,CAAC,CAAA;IAED,MAAM,OAAO,GAA2B,KAAK,EAAE,IAAI,EAAE,IAAI,EAAE,EAAE;QAC3D,MAAM,IAAI,GAAG,MAAM,CAAC,MAAM,IAAI,CAAe,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC,EAAE,CAAC,EAAE,SAAS,CAAC,IAAI,CAAC,EAAE,EAAE,IAAI,CAAC,WAAW,CAAC,CAAC,CAAC,CAAA;QAC7G,IAAI,IAAI,IAAI,IAAI;YAAE,MAAM,IAAI,gBAAgB,CAAC,GAAG,kBAAkB,CAAC,YAAY,YAAY,CAAC,CAAA;QAC5F,OAAO,EAAE,GAAG,IAAI,EAAE,CAAA;IACpB,CAAC,CAAA;IAED,OAAO,EAAE,IAAI,EAAE,OAAO,EAAE,CAAA;AAC1B,CAAC,CAAA"}
|
|
@@ -0,0 +1,33 @@
|
|
|
1
|
+
import { GoogleDriveKind } from '../consts.js';
|
|
2
|
+
/** The log scope of the package. */
|
|
3
|
+
export declare const LOG_SCOPE = "llm-reader-google-drive";
|
|
4
|
+
/** File downloads probed at once. */
|
|
5
|
+
export declare const DRIVE_PROBE_CONCURRENCY = 4;
|
|
6
|
+
/** Requests one probe may take: the download, then Drive's warning form (and its legacy link). */
|
|
7
|
+
export declare const DRIVE_PROBE_STEPS = 3;
|
|
8
|
+
/** Bytes of a probe's answer read — an interstitial or refusal page. */
|
|
9
|
+
export declare const DRIVE_PAGE_BYTES: number;
|
|
10
|
+
/** Bytes of one folder listing (a page of the embedded view, a Drive API page). */
|
|
11
|
+
export declare const DRIVE_LISTING_BYTES: number;
|
|
12
|
+
/** Drive API pages read per folder at most. */
|
|
13
|
+
export declare const DRIVE_API_MAX_PAGES = 20;
|
|
14
|
+
/**
|
|
15
|
+
* Drive serves files below this size without its "can't scan for viruses" form; an entry whose size
|
|
16
|
+
* is known and below it is not probed.
|
|
17
|
+
*/
|
|
18
|
+
export declare const DRIVE_SCAN_LIMIT_BYTES: number;
|
|
19
|
+
/** The media type prefix of Google's own item types — none has bytes of its own. */
|
|
20
|
+
export declare const GOOGLE_APPS_PREFIX = "application/vnd.google-apps.";
|
|
21
|
+
/** The shortcut type; its target is listed in its place. */
|
|
22
|
+
export declare const GOOGLE_APPS_SHORTCUT = "application/vnd.google-apps.shortcut";
|
|
23
|
+
/** The Google item types the source reads; every other `application/vnd.google-apps.*` is left out. */
|
|
24
|
+
export declare const GOOGLE_APPS_KINDS: Readonly<Record<string, GoogleDriveKind>>;
|
|
25
|
+
/** A refusal that means an exhausted download or API quota. */
|
|
26
|
+
export declare const QUOTA_PATTERN: RegExp;
|
|
27
|
+
/** A Drive API refusal of the key itself. */
|
|
28
|
+
export declare const API_KEY_PATTERN: RegExp;
|
|
29
|
+
/** A page that asks to sign in or to request access. */
|
|
30
|
+
export declare const NOT_PUBLIC_PATTERN: RegExp;
|
|
31
|
+
/** A refusal raised by a file-streams Drive resolver inside an injected fetch. */
|
|
32
|
+
export declare const RESOLVER_PATTERN: RegExp;
|
|
33
|
+
//# sourceMappingURL=consts.local.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"consts.local.d.ts","sourceRoot":"","sources":["../../src/source/consts.local.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,eAAe,EAAE,MAAM,cAAc,CAAA;AAE9C,oCAAoC;AACpC,eAAO,MAAM,SAAS,4BAA4B,CAAA;AAElD,qCAAqC;AACrC,eAAO,MAAM,uBAAuB,IAAI,CAAA;AAExC,kGAAkG;AAClG,eAAO,MAAM,iBAAiB,IAAI,CAAA;AAElC,wEAAwE;AACxE,eAAO,MAAM,gBAAgB,QAAa,CAAA;AAE1C,mFAAmF;AACnF,eAAO,MAAM,mBAAmB,QAAkB,CAAA;AAElD,+CAA+C;AAC/C,eAAO,MAAM,mBAAmB,KAAK,CAAA;AAErC;;;GAGG;AACH,eAAO,MAAM,sBAAsB,QAAoB,CAAA;AAEvD,oFAAoF;AACpF,eAAO,MAAM,kBAAkB,iCAAiC,CAAA;AAEhE,4DAA4D;AAC5D,eAAO,MAAM,oBAAoB,yCAAyC,CAAA;AAE1E,uGAAuG;AACvG,eAAO,MAAM,iBAAiB,EAAE,QAAQ,CAAC,MAAM,CAAC,MAAM,EAAE,eAAe,CAAC,CAMtE,CAAA;AAEF,+DAA+D;AAC/D,eAAO,MAAM,aAAa,QAAkI,CAAA;AAE5J,6CAA6C;AAC7C,eAAO,MAAM,eAAe,QAAyH,CAAA;AAErJ,wDAAwD;AACxD,eAAO,MAAM,kBAAkB,QAA8E,CAAA;AAE7G,kFAAkF;AAClF,eAAO,MAAM,gBAAgB,QAA0B,CAAA"}
|