@owlmeans/llm-reader-google-drive 0.0.0-stage → 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +113 -2
- package/agent-meta/manifest.json +16 -0
- package/agent-meta/skills/llm-reader-google-drive/SKILL.md +131 -0
- package/build/consts.d.ts +45 -0
- package/build/consts.d.ts.map +1 -0
- package/build/consts.js +54 -0
- package/build/consts.js.map +1 -0
- package/build/consts.local.d.ts +3 -0
- package/build/consts.local.d.ts.map +1 -0
- package/build/consts.local.js +3 -0
- package/build/consts.local.js.map +1 -0
- package/build/errors.d.ts +14 -0
- package/build/errors.d.ts.map +1 -0
- package/build/errors.js +26 -0
- package/build/errors.js.map +1 -0
- package/build/index.d.ts +6 -0
- package/build/index.d.ts.map +1 -0
- package/build/index.js +5 -0
- package/build/index.js.map +1 -0
- package/build/source/api.d.ts +9 -0
- package/build/source/api.d.ts.map +1 -0
- package/build/source/api.js +80 -0
- package/build/source/api.js.map +1 -0
- package/build/source/consts.local.d.ts +33 -0
- package/build/source/consts.local.d.ts.map +1 -0
- package/build/source/consts.local.js +39 -0
- package/build/source/consts.local.js.map +1 -0
- package/build/source/embedded.d.ts +10 -0
- package/build/source/embedded.d.ts.map +1 -0
- package/build/source/embedded.js +82 -0
- package/build/source/embedded.js.map +1 -0
- package/build/source/http.d.ts +6 -0
- package/build/source/http.d.ts.map +1 -0
- package/build/source/http.js +61 -0
- package/build/source/http.js.map +1 -0
- package/build/source/probe.d.ts +10 -0
- package/build/source/probe.d.ts.map +1 -0
- package/build/source/probe.js +80 -0
- package/build/source/probe.js.map +1 -0
- package/build/source/types.local.d.ts +87 -0
- package/build/source/types.local.d.ts.map +1 -0
- package/build/source/types.local.js +2 -0
- package/build/source/types.local.js.map +1 -0
- package/build/source.d.ts +22 -0
- package/build/source.d.ts.map +1 -0
- package/build/source.js +169 -0
- package/build/source.js.map +1 -0
- package/build/types.d.ts +79 -0
- package/build/types.d.ts.map +1 -0
- package/build/types.js +2 -0
- package/build/types.js.map +1 -0
- package/build/url/consts.local.d.ts +10 -0
- package/build/url/consts.local.d.ts.map +1 -0
- package/build/url/consts.local.js +15 -0
- package/build/url/consts.local.js.map +1 -0
- package/build/url.d.ts +8 -0
- package/build/url.d.ts.map +1 -0
- package/build/url.js +117 -0
- package/build/url.js.map +1 -0
- package/build/utils/names/types.d.ts +10 -0
- package/build/utils/names/types.d.ts.map +1 -0
- package/build/utils/names/types.js +2 -0
- package/build/utils/names/types.js.map +1 -0
- package/build/utils/names.d.ts +4 -0
- package/build/utils/names.d.ts.map +1 -0
- package/build/utils/names.js +31 -0
- package/build/utils/names.js.map +1 -0
- package/package.json +62 -4
- package/src/consts.local.ts +2 -0
- package/src/consts.ts +60 -0
- package/src/errors.ts +30 -0
- package/src/index.ts +7 -0
- package/src/source/api.ts +81 -0
- package/src/source/consts.local.ts +52 -0
- package/src/source/embedded.ts +83 -0
- package/src/source/http.ts +66 -0
- package/src/source/probe.ts +84 -0
- package/src/source/types.local.ts +91 -0
- package/src/source.ts +179 -0
- package/src/types.ts +86 -0
- package/src/url/consts.local.ts +19 -0
- package/src/url.ts +121 -0
- package/src/utils/names/types.ts +9 -0
- package/src/utils/names.ts +36 -0
- package/tests/context.ts +212 -0
- package/tests/fixtures.ts +88 -0
- package/tests/live.spec.ts +53 -0
- package/tests/source.spec.ts +189 -0
- package/tests/tsconfig.json +12 -0
- package/tsconfig.json +16 -0
|
@@ -0,0 +1,39 @@
|
|
|
1
|
+
import { GoogleDriveKind } from '../consts.js';
|
|
2
|
+
/** The log scope of the package. */
|
|
3
|
+
export const LOG_SCOPE = 'llm-reader-google-drive';
|
|
4
|
+
/** File downloads probed at once. */
|
|
5
|
+
export const DRIVE_PROBE_CONCURRENCY = 4;
|
|
6
|
+
/** Requests one probe may take: the download, then Drive's warning form (and its legacy link). */
|
|
7
|
+
export const DRIVE_PROBE_STEPS = 3;
|
|
8
|
+
/** Bytes of a probe's answer read — an interstitial or refusal page. */
|
|
9
|
+
export const DRIVE_PAGE_BYTES = 256 * 1024;
|
|
10
|
+
/** Bytes of one folder listing (a page of the embedded view, a Drive API page). */
|
|
11
|
+
export const DRIVE_LISTING_BYTES = 4 * 1024 * 1024;
|
|
12
|
+
/** Drive API pages read per folder at most. */
|
|
13
|
+
export const DRIVE_API_MAX_PAGES = 20;
|
|
14
|
+
/**
|
|
15
|
+
* Drive serves files below this size without its "can't scan for viruses" form; an entry whose size
|
|
16
|
+
* is known and below it is not probed.
|
|
17
|
+
*/
|
|
18
|
+
export const DRIVE_SCAN_LIMIT_BYTES = 100 * 1024 * 1024;
|
|
19
|
+
/** The media type prefix of Google's own item types — none has bytes of its own. */
|
|
20
|
+
export const GOOGLE_APPS_PREFIX = 'application/vnd.google-apps.';
|
|
21
|
+
/** The shortcut type; its target is listed in its place. */
|
|
22
|
+
export const GOOGLE_APPS_SHORTCUT = 'application/vnd.google-apps.shortcut';
|
|
23
|
+
/** The Google item types the source reads; every other `application/vnd.google-apps.*` is left out. */
|
|
24
|
+
export const GOOGLE_APPS_KINDS = Object.freeze({
|
|
25
|
+
'application/vnd.google-apps.folder': GoogleDriveKind.Folder,
|
|
26
|
+
'application/vnd.google-apps.document': GoogleDriveKind.Document,
|
|
27
|
+
'application/vnd.google-apps.spreadsheet': GoogleDriveKind.Spreadsheet,
|
|
28
|
+
'application/vnd.google-apps.presentation': GoogleDriveKind.Presentation,
|
|
29
|
+
'application/vnd.google-apps.drawing': GoogleDriveKind.Drawing,
|
|
30
|
+
});
|
|
31
|
+
/** A refusal that means an exhausted download or API quota. */
|
|
32
|
+
export const QUOTA_PATTERN = /too many users have viewed or downloaded|quota|dailyLimitExceeded|userRateLimitExceeded|rateLimitExceeded|RESOURCE_EXHAUSTED/i;
|
|
33
|
+
/** A Drive API refusal of the key itself. */
|
|
34
|
+
export const API_KEY_PATTERN = /API_KEY_INVALID|API key not valid|keyInvalid|keyExpired|accessNotConfigured|SERVICE_DISABLED|API_KEY_SERVICE_BLOCKED/;
|
|
35
|
+
/** A page that asks to sign in or to request access. */
|
|
36
|
+
export const NOT_PUBLIC_PATTERN = /ServiceLogin|\/v3\/signin|you need access|request access|need permission/i;
|
|
37
|
+
/** A refusal raised by a file-streams Drive resolver inside an injected fetch. */
|
|
38
|
+
export const RESOLVER_PATTERN = /resolver:google-drive/;
|
|
39
|
+
//# sourceMappingURL=consts.local.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"consts.local.js","sourceRoot":"","sources":["../../src/source/consts.local.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,eAAe,EAAE,MAAM,cAAc,CAAA;AAE9C,oCAAoC;AACpC,MAAM,CAAC,MAAM,SAAS,GAAG,yBAAyB,CAAA;AAElD,qCAAqC;AACrC,MAAM,CAAC,MAAM,uBAAuB,GAAG,CAAC,CAAA;AAExC,kGAAkG;AAClG,MAAM,CAAC,MAAM,iBAAiB,GAAG,CAAC,CAAA;AAElC,wEAAwE;AACxE,MAAM,CAAC,MAAM,gBAAgB,GAAG,GAAG,GAAG,IAAI,CAAA;AAE1C,mFAAmF;AACnF,MAAM,CAAC,MAAM,mBAAmB,GAAG,CAAC,GAAG,IAAI,GAAG,IAAI,CAAA;AAElD,+CAA+C;AAC/C,MAAM,CAAC,MAAM,mBAAmB,GAAG,EAAE,CAAA;AAErC;;;GAGG;AACH,MAAM,CAAC,MAAM,sBAAsB,GAAG,GAAG,GAAG,IAAI,GAAG,IAAI,CAAA;AAEvD,oFAAoF;AACpF,MAAM,CAAC,MAAM,kBAAkB,GAAG,8BAA8B,CAAA;AAEhE,4DAA4D;AAC5D,MAAM,CAAC,MAAM,oBAAoB,GAAG,sCAAsC,CAAA;AAE1E,uGAAuG;AACvG,MAAM,CAAC,MAAM,iBAAiB,GAA8C,MAAM,CAAC,MAAM,CAAC;IACxF,oCAAoC,EAAE,eAAe,CAAC,MAAM;IAC5D,sCAAsC,EAAE,eAAe,CAAC,QAAQ;IAChE,yCAAyC,EAAE,eAAe,CAAC,WAAW;IACtE,0CAA0C,EAAE,eAAe,CAAC,YAAY;IACxE,qCAAqC,EAAE,eAAe,CAAC,OAAO;CAC/D,CAAC,CAAA;AAEF,+DAA+D;AAC/D,MAAM,CAAC,MAAM,aAAa,GAAG,+HAA+H,CAAA;AAE5J,6CAA6C;AAC7C,MAAM,CAAC,MAAM,eAAe,GAAG,sHAAsH,CAAA;AAErJ,wDAAwD;AACxD,MAAM,CAAC,MAAM,kBAAkB,GAAG,2EAA2E,CAAA;AAE7G,kFAAkF;AAClF,MAAM,CAAC,MAAM,gBAAgB,GAAG,uBAAuB,CAAA"}
|
|
@@ -0,0 +1,10 @@
|
|
|
1
|
+
import type { GoogleDriveUrlHelper } from '../types.js';
|
|
2
|
+
import type { DriveHttp, DriveLister } from './types.local.js';
|
|
3
|
+
/**
|
|
4
|
+
* Folders listed without a key, from Drive's static embedded folder view
|
|
5
|
+
* (`/embeddedfolderview?id=<id>`): every `div.flip-entry` (`id="entry-<fileId>"`) carries a link
|
|
6
|
+
* (`/file/d/<id>/view` for a file, `/drive/folders/<id>` for a sub-folder, `docs.google.com/…` for an
|
|
7
|
+
* editor file), its `.flip-entry-title` and a type icon (`…/type/<media type>`). No sizes.
|
|
8
|
+
*/
|
|
9
|
+
export declare const makeEmbeddedLister: (http: DriveHttp, urls: GoogleDriveUrlHelper) => DriveLister;
|
|
10
|
+
//# sourceMappingURL=embedded.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"embedded.d.ts","sourceRoot":"","sources":["../../src/source/embedded.ts"],"names":[],"mappings":"AAKA,OAAO,KAAK,EAAE,oBAAoB,EAAE,MAAM,aAAa,CAAA;AAEvD,OAAO,KAAK,EAAE,SAAS,EAAa,WAAW,EAAE,MAAM,kBAAkB,CAAA;AAIzE;;;;;GAKG;AACH,eAAO,MAAM,kBAAkB,SAAU,SAAS,QAAQ,oBAAoB,KAAG,WAiEhF,CAAA"}
|
|
@@ -0,0 +1,82 @@
|
|
|
1
|
+
import { logger } from '@owlmeans/log';
|
|
2
|
+
import { parseHTML } from 'linkedom';
|
|
3
|
+
import { GoogleDriveFailure, GoogleDriveKind } from '../consts.js';
|
|
4
|
+
import { DRIVE_ID_PATTERN } from '../consts.local.js';
|
|
5
|
+
import { GoogleDriveError } from '../errors.js';
|
|
6
|
+
import { DRIVE_LISTING_BYTES, GOOGLE_APPS_KINDS, GOOGLE_APPS_PREFIX, LOG_SCOPE } from './consts.local.js';
|
|
7
|
+
const log = logger(LOG_SCOPE);
|
|
8
|
+
/**
|
|
9
|
+
* Folders listed without a key, from Drive's static embedded folder view
|
|
10
|
+
* (`/embeddedfolderview?id=<id>`): every `div.flip-entry` (`id="entry-<fileId>"`) carries a link
|
|
11
|
+
* (`/file/d/<id>/view` for a file, `/drive/folders/<id>` for a sub-folder, `docs.google.com/…` for an
|
|
12
|
+
* editor file), its `.flip-entry-title` and a type icon (`…/type/<media type>`). No sizes.
|
|
13
|
+
*/
|
|
14
|
+
export const makeEmbeddedLister = (http, urls) => {
|
|
15
|
+
const mimeOf = (src, base) => {
|
|
16
|
+
if (src == null || src === '')
|
|
17
|
+
return undefined;
|
|
18
|
+
try {
|
|
19
|
+
const type = /\/type\/(.+)$/.exec(new URL(src, base).pathname)?.[1];
|
|
20
|
+
return type != null ? decodeURIComponent(type) : undefined;
|
|
21
|
+
}
|
|
22
|
+
catch {
|
|
23
|
+
return undefined;
|
|
24
|
+
}
|
|
25
|
+
};
|
|
26
|
+
const list = async (folder) => {
|
|
27
|
+
const response = await http.get(urls.folderViewUrl(folder.id, folder.resourceKey), { maxBytes: DRIVE_LISTING_BYTES });
|
|
28
|
+
const refused = await http.refusal(response);
|
|
29
|
+
if (refused != null)
|
|
30
|
+
throw refused;
|
|
31
|
+
const base = response.url !== '' ? response.url : urls.folderViewUrl(folder.id);
|
|
32
|
+
const page = await response.text();
|
|
33
|
+
const { document } = parseHTML(page);
|
|
34
|
+
if (document.querySelector('#flip-contents, .flip-entries, .flip-entry') == null) {
|
|
35
|
+
throw http.pageFailure(page) ?? new GoogleDriveError(`${GoogleDriveFailure.Unrecognized}:folder-view`);
|
|
36
|
+
}
|
|
37
|
+
const items = [];
|
|
38
|
+
let unsupported = 0;
|
|
39
|
+
for (const entry of document.querySelectorAll('.flip-entry')) {
|
|
40
|
+
const href = entry.querySelector('a[href]')?.getAttribute('href');
|
|
41
|
+
let link;
|
|
42
|
+
try {
|
|
43
|
+
link = href != null ? urls.parse(new URL(href, base).href) : undefined;
|
|
44
|
+
}
|
|
45
|
+
catch {
|
|
46
|
+
link = undefined;
|
|
47
|
+
}
|
|
48
|
+
const mime = mimeOf(entry.querySelector('.flip-entry-list-icon img, .flip-entry-icon img')?.getAttribute('src'), base);
|
|
49
|
+
const fallback = /^entry-(.+)$/.exec(entry.getAttribute('id') ?? '')?.[1];
|
|
50
|
+
const id = link?.id ?? (fallback != null && DRIVE_ID_PATTERN.test(fallback) ? fallback : undefined);
|
|
51
|
+
const kind = link != null && link.kind !== GoogleDriveKind.Unresolved ? link.kind
|
|
52
|
+
: mime != null && mime.startsWith(GOOGLE_APPS_PREFIX) ? GOOGLE_APPS_KINDS[mime]
|
|
53
|
+
: id != null ? GoogleDriveKind.File : undefined;
|
|
54
|
+
if (id == null || kind == null) {
|
|
55
|
+
unsupported++;
|
|
56
|
+
continue;
|
|
57
|
+
}
|
|
58
|
+
const title = entry.querySelector('.flip-entry-title')?.textContent?.trim();
|
|
59
|
+
items.push({
|
|
60
|
+
id, kind, name: title != null && title !== '' ? title : id,
|
|
61
|
+
...(mime != null && !mime.startsWith(GOOGLE_APPS_PREFIX) ? { mime } : {}),
|
|
62
|
+
...(link?.resourceKey != null ? { resourceKey: link.resourceKey } : {}),
|
|
63
|
+
});
|
|
64
|
+
}
|
|
65
|
+
log.debug('Listed a folder', { folder: folder.id, items: items.length, unsupported });
|
|
66
|
+
return items;
|
|
67
|
+
};
|
|
68
|
+
const resolve = async (url, link) => {
|
|
69
|
+
// `open?id=` answers with a redirect to the item's own page; only where it lands is read.
|
|
70
|
+
const response = await http.get(url, { maxBytes: DRIVE_LISTING_BYTES });
|
|
71
|
+
const refused = await http.refusal(response);
|
|
72
|
+
if (refused != null)
|
|
73
|
+
throw refused;
|
|
74
|
+
await http.discard(response);
|
|
75
|
+
const landed = response.url !== '' ? urls.parse(response.url) : undefined;
|
|
76
|
+
const resourceKey = landed?.resourceKey ?? link.resourceKey;
|
|
77
|
+
const kind = landed != null && landed.kind !== GoogleDriveKind.Unresolved ? landed.kind : GoogleDriveKind.File;
|
|
78
|
+
return { kind, id: landed?.id ?? link.id, ...(resourceKey != null ? { resourceKey } : {}) };
|
|
79
|
+
};
|
|
80
|
+
return { list, resolve };
|
|
81
|
+
};
|
|
82
|
+
//# sourceMappingURL=embedded.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"embedded.js","sourceRoot":"","sources":["../../src/source/embedded.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,MAAM,EAAE,MAAM,eAAe,CAAA;AACtC,OAAO,EAAE,SAAS,EAAE,MAAM,UAAU,CAAA;AACpC,OAAO,EAAE,kBAAkB,EAAE,eAAe,EAAE,MAAM,cAAc,CAAA;AAClE,OAAO,EAAE,gBAAgB,EAAE,MAAM,oBAAoB,CAAA;AACrD,OAAO,EAAE,gBAAgB,EAAE,MAAM,cAAc,CAAA;AAE/C,OAAO,EAAE,mBAAmB,EAAE,iBAAiB,EAAE,kBAAkB,EAAE,SAAS,EAAE,MAAM,mBAAmB,CAAA;AAGzG,MAAM,GAAG,GAAG,MAAM,CAAC,SAAS,CAAC,CAAA;AAE7B;;;;;GAKG;AACH,MAAM,CAAC,MAAM,kBAAkB,GAAG,CAAC,IAAe,EAAE,IAA0B,EAAe,EAAE;IAC7F,MAAM,MAAM,GAAG,CAAC,GAA8B,EAAE,IAAY,EAAsB,EAAE;QAClF,IAAI,GAAG,IAAI,IAAI,IAAI,GAAG,KAAK,EAAE;YAAE,OAAO,SAAS,CAAA;QAC/C,IAAI,CAAC;YACH,MAAM,IAAI,GAAG,eAAe,CAAC,IAAI,CAAC,IAAI,GAAG,CAAC,GAAG,EAAE,IAAI,CAAC,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC,CAAC,CAAA;YACnE,OAAO,IAAI,IAAI,IAAI,CAAC,CAAC,CAAC,kBAAkB,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,SAAS,CAAA;QAC5D,CAAC;QAAC,MAAM,CAAC;YACP,OAAO,SAAS,CAAA;QAClB,CAAC;IACH,CAAC,CAAA;IAED,MAAM,IAAI,GAAwB,KAAK,EAAC,MAAM,EAAC,EAAE;QAC/C,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,aAAa,CAAC,MAAM,CAAC,EAAE,EAAE,MAAM,CAAC,WAAW,CAAC,EAAE,EAAE,QAAQ,EAAE,mBAAmB,EAAE,CAAC,CAAA;QACrH,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAA;QAC5C,IAAI,OAAO,IAAI,IAAI;YAAE,MAAM,OAAO,CAAA;QAClC,MAAM,IAAI,GAAG,QAAQ,CAAC,GAAG,KAAK,EAAE,CAAC,CAAC,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,aAAa,CAAC,MAAM,CAAC,EAAE,CAAC,CAAA;QAC/E,MAAM,IAAI,GAAG,MAAM,QAAQ,CAAC,IAAI,EAAE,CAAA;QAClC,MAAM,EAAE,QAAQ,EAAE,GAAG,SAAS,CAAC,IAAI,CAAC,CAAA;QACpC,IAAI,QAAQ,CAAC,aAAa,CAAC,4CAA4C,CAAC,IAAI,IAAI,EAAE,CAAC;YACjF,MAAM,IAAI,CAAC,WAAW,CAAC,IAAI,CAAC,IAAI,IAAI,gBAAgB,CAAC,GAAG,kBAAkB,CAAC,YAAY,cAAc,CAAC,CAAA;QACxG,CAAC;QACD,MAAM,KAAK,GAAgB,EAAE,CAAA;QAC7B,IAAI,WAAW,GAAG,CAAC,CAAA;QACnB,KAAK,MAAM,KAAK,IAAI,QAAQ,CAAC,gBAAgB,CAAC,aAAa,CAAC,EAAE,CAAC;YAC7D,MAAM,IAAI,GAAG,KAAK,CAAC,aAAa,CAAC,SAAS,CAAC,EAAE,YAAY,CAAC,MAAM,CAAC,CAAA;YACjE,IAAI,IAAI,CAAA;YACR,IAAI,CAAC;gBACH,IAAI,GAAG,IAAI,IAAI,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,GAAG,CAAC,IAAI,EAAE,IAAI,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,SAAS,CAAA;YACxE,CAAC;YAAC,MAAM,CAAC;gBACP,IAAI,GAAG,SAAS,CAAA;YAClB,CAAC;YACD,MAAM,IAAI,GAAG,MAAM,CAAC,KAAK,CAAC,aAAa,CAAC,iDAAiD,CAAC,EAAE,YAAY,CAAC,KAAK,CAAC,EAAE,IAAI,CAAC,CAAA;YACtH,MAAM,QAAQ,GAAG,cAAc,CAAC,IAAI,CAAC,KAAK,CAAC,YAAY,CAAC,IAAI,CAAC,IAAI,EAAE,CAAC,EAAE,CAAC,CAAC,CAAC,CAAA;YACzE,MAAM,EAAE,GAAG,IAAI,EAAE,EAAE,IAAI,CAAC,QAAQ,IAAI,IAAI,IAAI,gBAAgB,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,SAAS,CAAC,CAAA;YACnG,MAAM,IAAI,GAAG,IAAI,IAAI,IAAI,IAAI,IAAI,CAAC,IAAI,KAAK,eAAe,CAAC,UAAU,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI;gBAC/E,CAAC,CAAC,IAAI,IAAI,IAAI,IAAI,IAAI,CAAC,UAAU,CAAC,kBAAkB,CAAC,CAAC,CAAC,CAAC,iBAAiB,CAAC,IAAI,CAAC;oBAC7E,CAAC,CAAC,EAAE,IAAI,IAAI,CAAC,CAAC,CAAC,eAAe,CAAC,IAAI,CAAC,CAAC,CAAC,SAAS,CAAA;YACnD,IAAI,EAAE,IAAI,IAAI,IAAI,IAAI,IAAI,IAAI,EAAE,CAAC;gBAC/B,WAAW,EAAE,CAAA;gBACb,SAAQ;YACV,CAAC;YACD,MAAM,KAAK,GAAG,KAAK,CAAC,aAAa,CAAC,mBAAmB,CAAC,EAAE,WAAW,EAAE,IAAI,EAAE,CAAA;YAC3E,KAAK,CAAC,IAAI,CAAC;gBACT,EAAE,EAAE,IAAI,EAAE,IAAI,EAAE,KAAK,IAAI,IAAI,IAAI,KAAK,KAAK,EAAE,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE;gBAC1D,GAAG,CAAC,IAAI,IAAI,IAAI,IAAI,CAAC,IAAI,CAAC,UAAU,CAAC,kBAAkB,CAAC,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;gBACzE,GAAG,CAAC,IAAI,EAAE,WAAW,IAAI,IAAI,CAAC,CAAC,CAAC,EAAE,WAAW,EAAE,IAAI,CAAC,WAAW,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;aACxE,CAAC,CAAA;QACJ,CAAC;QACD,GAAG,CAAC,KAAK,CAAC,iBAAiB,EAAE,EAAE,MAAM,EAAE,MAAM,CAAC,EAAE,EAAE,KAAK,EAAE,KAAK,CAAC,MAAM,EAAE,WAAW,EAAE,CAAC,CAAA;QACrF,OAAO,KAAK,CAAA;IACd,CAAC,CAAA;IAED,MAAM,OAAO,GAA2B,KAAK,EAAE,GAAG,EAAE,IAAI,EAAE,EAAE;QAC1D,0FAA0F;QAC1F,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,GAAG,CAAC,GAAG,EAAE,EAAE,QAAQ,EAAE,mBAAmB,EAAE,CAAC,CAAA;QACvE,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAA;QAC5C,IAAI,OAAO,IAAI,IAAI;YAAE,MAAM,OAAO,CAAA;QAClC,MAAM,IAAI,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAA;QAC5B,MAAM,MAAM,GAAG,QAAQ,CAAC,GAAG,KAAK,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,SAAS,CAAA;QACzE,MAAM,WAAW,GAAG,MAAM,EAAE,WAAW,IAAI,IAAI,CAAC,WAAW,CAAA;QAC3D,MAAM,IAAI,GAAG,MAAM,IAAI,IAAI,IAAI,MAAM,CAAC,IAAI,KAAK,eAAe,CAAC,UAAU,CAAC,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,eAAe,CAAC,IAAI,CAAA;QAC9G,OAAO,EAAE,IAAI,EAAE,EAAE,EAAE,MAAM,EAAE,EAAE,IAAI,IAAI,CAAC,EAAE,EAAE,GAAG,CAAC,WAAW,IAAI,IAAI,CAAC,CAAC,CAAC,EAAE,WAAW,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,CAAA;IAC7F,CAAC,CAAA;IAED,OAAO,EAAE,IAAI,EAAE,OAAO,EAAE,CAAA;AAC1B,CAAC,CAAA"}
|
|
@@ -0,0 +1,6 @@
|
|
|
1
|
+
import type { ReaderSourceContext } from '@owlmeans/llm-reader';
|
|
2
|
+
import type { GoogleDriveUrlHelper } from '../types.js';
|
|
3
|
+
import type { DriveHttp } from './types.local.js';
|
|
4
|
+
/** Google, asked through one collect's guarded fetch — its answers read as Drive refusals. */
|
|
5
|
+
export declare const makeDriveHttp: (context: ReaderSourceContext, urls: GoogleDriveUrlHelper) => DriveHttp;
|
|
6
|
+
//# sourceMappingURL=http.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"http.d.ts","sourceRoot":"","sources":["../../src/source/http.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,mBAAmB,EAAE,MAAM,sBAAsB,CAAA;AAG/D,OAAO,KAAK,EAAE,oBAAoB,EAAE,MAAM,aAAa,CAAA;AAEvD,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,kBAAkB,CAAA;AAEjD,8FAA8F;AAC9F,eAAO,MAAM,aAAa,YAAa,mBAAmB,QAAQ,oBAAoB,KAAG,SAyDxF,CAAA"}
|
|
@@ -0,0 +1,61 @@
|
|
|
1
|
+
import { GoogleDriveFailure } from '../consts.js';
|
|
2
|
+
import { GoogleDriveError } from '../errors.js';
|
|
3
|
+
import { API_KEY_PATTERN, NOT_PUBLIC_PATTERN, QUOTA_PATTERN, RESOLVER_PATTERN } from './consts.local.js';
|
|
4
|
+
/** Google, asked through one collect's guarded fetch — its answers read as Drive refusals. */
|
|
5
|
+
export const makeDriveHttp = (context, urls) => {
|
|
6
|
+
const fail = (reason) => new GoogleDriveError(reason);
|
|
7
|
+
const messagesOf = (error) => {
|
|
8
|
+
const parts = [];
|
|
9
|
+
for (let current = error, depth = 0; current != null && depth < 4; current = current.cause, depth++) {
|
|
10
|
+
parts.push(String(current.message ?? current));
|
|
11
|
+
}
|
|
12
|
+
return parts.join(' ');
|
|
13
|
+
};
|
|
14
|
+
const get = async (url, init) => {
|
|
15
|
+
try {
|
|
16
|
+
return await context.fetch(url, init);
|
|
17
|
+
}
|
|
18
|
+
catch (error) {
|
|
19
|
+
if (context.signal?.aborted !== true && RESOLVER_PATTERN.test(messagesOf(error))) {
|
|
20
|
+
throw Object.assign(fail(GoogleDriveFailure.Resolver), { cause: error, url });
|
|
21
|
+
}
|
|
22
|
+
throw error;
|
|
23
|
+
}
|
|
24
|
+
};
|
|
25
|
+
const discard = async (response) => {
|
|
26
|
+
await response.body?.cancel().catch(() => undefined);
|
|
27
|
+
};
|
|
28
|
+
const text = async (response) => {
|
|
29
|
+
try {
|
|
30
|
+
return await response.text();
|
|
31
|
+
}
|
|
32
|
+
catch {
|
|
33
|
+
return '';
|
|
34
|
+
}
|
|
35
|
+
};
|
|
36
|
+
const pageFailure = page => {
|
|
37
|
+
if (QUOTA_PATTERN.test(page))
|
|
38
|
+
return fail(GoogleDriveFailure.Quota);
|
|
39
|
+
if (NOT_PUBLIC_PATTERN.test(page))
|
|
40
|
+
return fail(GoogleDriveFailure.NotPublic);
|
|
41
|
+
return undefined;
|
|
42
|
+
};
|
|
43
|
+
const refusal = async (response) => {
|
|
44
|
+
if (response.url !== '' && urls.isSignIn(response.url)) {
|
|
45
|
+
await discard(response);
|
|
46
|
+
return fail(GoogleDriveFailure.NotPublic);
|
|
47
|
+
}
|
|
48
|
+
if (response.status < 400)
|
|
49
|
+
return undefined;
|
|
50
|
+
const body = await text(response);
|
|
51
|
+
const error = response.status === 429 || QUOTA_PATTERN.test(body) ? fail(GoogleDriveFailure.Quota)
|
|
52
|
+
: API_KEY_PATTERN.test(body) ? fail(GoogleDriveFailure.ApiKey)
|
|
53
|
+
: response.status === 404 ? fail(GoogleDriveFailure.NotFound)
|
|
54
|
+
: response.status === 401 || response.status === 403 ? fail(GoogleDriveFailure.NotPublic)
|
|
55
|
+
: fail(`${GoogleDriveFailure.Status}:${response.status}`);
|
|
56
|
+
error.status = response.status;
|
|
57
|
+
return error;
|
|
58
|
+
};
|
|
59
|
+
return { get, refusal, pageFailure, text, discard };
|
|
60
|
+
};
|
|
61
|
+
//# sourceMappingURL=http.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"http.js","sourceRoot":"","sources":["../../src/source/http.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,kBAAkB,EAAE,MAAM,cAAc,CAAA;AACjD,OAAO,EAAE,gBAAgB,EAAE,MAAM,cAAc,CAAA;AAE/C,OAAO,EAAE,eAAe,EAAE,kBAAkB,EAAE,aAAa,EAAE,gBAAgB,EAAE,MAAM,mBAAmB,CAAA;AAGxG,8FAA8F;AAC9F,MAAM,CAAC,MAAM,aAAa,GAAG,CAAC,OAA4B,EAAE,IAA0B,EAAa,EAAE;IACnG,MAAM,IAAI,GAAG,CAAC,MAAc,EAAoB,EAAE,CAAC,IAAI,gBAAgB,CAAC,MAAM,CAAC,CAAA;IAE/E,MAAM,UAAU,GAAG,CAAC,KAAc,EAAU,EAAE;QAC5C,MAAM,KAAK,GAAa,EAAE,CAAA;QAC1B,KAAK,IAAI,OAAO,GAAG,KAAK,EAAE,KAAK,GAAG,CAAC,EAAE,OAAO,IAAI,IAAI,IAAI,KAAK,GAAG,CAAC,EAAE,OAAO,GAAI,OAA+B,CAAC,KAAK,EAAE,KAAK,EAAE,EAAE,CAAC;YAC7H,KAAK,CAAC,IAAI,CAAC,MAAM,CAAE,OAAiC,CAAC,OAAO,IAAI,OAAO,CAAC,CAAC,CAAA;QAC3E,CAAC;QACD,OAAO,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,CAAA;IACxB,CAAC,CAAA;IAED,MAAM,GAAG,GAAqB,KAAK,EAAE,GAAG,EAAE,IAAI,EAAE,EAAE;QAChD,IAAI,CAAC;YACH,OAAO,MAAM,OAAO,CAAC,KAAK,CAAC,GAAG,EAAE,IAAI,CAAC,CAAA;QACvC,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,IAAI,OAAO,CAAC,MAAM,EAAE,OAAO,KAAK,IAAI,IAAI,gBAAgB,CAAC,IAAI,CAAC,UAAU,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC;gBACjF,MAAM,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,kBAAkB,CAAC,QAAQ,CAAC,EAAE,EAAE,KAAK,EAAE,KAAK,EAAE,GAAG,EAAE,CAAC,CAAA;YAC/E,CAAC;YACD,MAAM,KAAK,CAAA;QACb,CAAC;IACH,CAAC,CAAA;IAED,MAAM,OAAO,GAAyB,KAAK,EAAC,QAAQ,EAAC,EAAE;QACrD,MAAM,QAAQ,CAAC,IAAI,EAAE,MAAM,EAAE,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,SAAS,CAAC,CAAA;IACtD,CAAC,CAAA;IAED,MAAM,IAAI,GAAsB,KAAK,EAAC,QAAQ,EAAC,EAAE;QAC/C,IAAI,CAAC;YACH,OAAO,MAAM,QAAQ,CAAC,IAAI,EAAE,CAAA;QAC9B,CAAC;QAAC,MAAM,CAAC;YACP,OAAO,EAAE,CAAA;QACX,CAAC;IACH,CAAC,CAAA;IAED,MAAM,WAAW,GAA6B,IAAI,CAAC,EAAE;QACnD,IAAI,aAAa,CAAC,IAAI,CAAC,IAAI,CAAC;YAAE,OAAO,IAAI,CAAC,kBAAkB,CAAC,KAAK,CAAC,CAAA;QACnE,IAAI,kBAAkB,CAAC,IAAI,CAAC,IAAI,CAAC;YAAE,OAAO,IAAI,CAAC,kBAAkB,CAAC,SAAS,CAAC,CAAA;QAC5E,OAAO,SAAS,CAAA;IAClB,CAAC,CAAA;IAED,MAAM,OAAO,GAAyB,KAAK,EAAC,QAAQ,EAAC,EAAE;QACrD,IAAI,QAAQ,CAAC,GAAG,KAAK,EAAE,IAAI,IAAI,CAAC,QAAQ,CAAC,QAAQ,CAAC,GAAG,CAAC,EAAE,CAAC;YACvD,MAAM,OAAO,CAAC,QAAQ,CAAC,CAAA;YACvB,OAAO,IAAI,CAAC,kBAAkB,CAAC,SAAS,CAAC,CAAA;QAC3C,CAAC;QACD,IAAI,QAAQ,CAAC,MAAM,GAAG,GAAG;YAAE,OAAO,SAAS,CAAA;QAC3C,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,QAAQ,CAAC,CAAA;QACjC,MAAM,KAAK,GAAG,QAAQ,CAAC,MAAM,KAAK,GAAG,IAAI,aAAa,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,kBAAkB,CAAC,KAAK,CAAC;YAChG,CAAC,CAAC,eAAe,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,kBAAkB,CAAC,MAAM,CAAC;gBAC5D,CAAC,CAAC,QAAQ,CAAC,MAAM,KAAK,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,kBAAkB,CAAC,QAAQ,CAAC;oBAC3D,CAAC,CAAC,QAAQ,CAAC,MAAM,KAAK,GAAG,IAAI,QAAQ,CAAC,MAAM,KAAK,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,kBAAkB,CAAC,SAAS,CAAC;wBACvF,CAAC,CAAC,IAAI,CAAC,GAAG,kBAAkB,CAAC,MAAM,IAAI,QAAQ,CAAC,MAAM,EAAE,CAAC,CAAA;QACjE,KAAK,CAAC,MAAM,GAAG,QAAQ,CAAC,MAAM,CAAA;QAC9B,OAAO,KAAK,CAAA;IACd,CAAC,CAAA;IAED,OAAO,EAAE,GAAG,EAAE,OAAO,EAAE,WAAW,EAAE,IAAI,EAAE,OAAO,EAAE,CAAA;AACrD,CAAC,CAAA"}
|
|
@@ -0,0 +1,10 @@
|
|
|
1
|
+
import type { GoogleDriveUrlHelper } from '../types.js';
|
|
2
|
+
import type { DriveHttp, DriveProbe } from './types.local.js';
|
|
3
|
+
/**
|
|
4
|
+
* Downloads asked before the collector takes them: one ranged GET (`bytes=0-0`) tells the name
|
|
5
|
+
* (`Content-Disposition`) and size (`Content-Range`) of a file Drive serves as an attachment; an HTML
|
|
6
|
+
* answer is Drive's "can't scan for viruses" form — followed by its action and hidden inputs, like a
|
|
7
|
+
* browser would — or a refusal (sign-in, quota).
|
|
8
|
+
*/
|
|
9
|
+
export declare const makeDriveProbe: (http: DriveHttp, urls: GoogleDriveUrlHelper) => DriveProbe;
|
|
10
|
+
//# sourceMappingURL=probe.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"probe.d.ts","sourceRoot":"","sources":["../../src/source/probe.ts"],"names":[],"mappings":"AAGA,OAAO,KAAK,EAAE,oBAAoB,EAAE,MAAM,aAAa,CAAA;AAGvD,OAAO,KAAK,EAAE,SAAS,EAAE,UAAU,EAAc,MAAM,kBAAkB,CAAA;AAEzE;;;;;GAKG;AACH,eAAO,MAAM,cAAc,SAAU,SAAS,QAAQ,oBAAoB,KAAG,UAqE5E,CAAA"}
|
|
@@ -0,0 +1,80 @@
|
|
|
1
|
+
import { parseHTML } from 'linkedom';
|
|
2
|
+
import { GoogleDriveFailure } from '../consts.js';
|
|
3
|
+
import { GoogleDriveError } from '../errors.js';
|
|
4
|
+
import { nameUtils } from '../utils/names.js';
|
|
5
|
+
import { DRIVE_PAGE_BYTES, DRIVE_PROBE_STEPS } from './consts.local.js';
|
|
6
|
+
/**
|
|
7
|
+
* Downloads asked before the collector takes them: one ranged GET (`bytes=0-0`) tells the name
|
|
8
|
+
* (`Content-Disposition`) and size (`Content-Range`) of a file Drive serves as an attachment; an HTML
|
|
9
|
+
* answer is Drive's "can't scan for viruses" form — followed by its action and hidden inputs, like a
|
|
10
|
+
* browser would — or a refusal (sign-in, quota).
|
|
11
|
+
*/
|
|
12
|
+
export const makeDriveProbe = (http, urls) => {
|
|
13
|
+
const isAttachment = (response) => /attachment/i.test(response.headers.get('content-disposition') ?? '') || !/text\/html/i.test(response.headers.get('content-type') ?? '');
|
|
14
|
+
const sizeOf = (response) => {
|
|
15
|
+
const total = Number(/\/(\d+)\s*$/.exec(response.headers.get('content-range') ?? '')?.[1] ?? NaN);
|
|
16
|
+
if (Number.isFinite(total))
|
|
17
|
+
return total;
|
|
18
|
+
const length = Number(response.headers.get('content-length') ?? NaN);
|
|
19
|
+
return response.status === 200 && Number.isFinite(length) ? length : undefined;
|
|
20
|
+
};
|
|
21
|
+
const described = (url, response, size = true) => {
|
|
22
|
+
const name = nameUtils.fromDisposition(response.headers.get('content-disposition'));
|
|
23
|
+
const bytes = size ? sizeOf(response) : undefined;
|
|
24
|
+
return { url, ...(name != null ? { name } : {}), ...(bytes != null ? { size: bytes } : {}) };
|
|
25
|
+
};
|
|
26
|
+
/** The URL Drive's warning page sends a browser on to — its form with the hidden inputs, or the legacy confirm link. */
|
|
27
|
+
const followUp = (page, base) => {
|
|
28
|
+
const { document } = parseHTML(page);
|
|
29
|
+
const form = document.querySelector('form#download-form') ?? document.querySelector('form[action*="download"]');
|
|
30
|
+
const action = form?.getAttribute('action');
|
|
31
|
+
if (form != null && action != null && action !== '') {
|
|
32
|
+
const next = new URL(action, base);
|
|
33
|
+
for (const input of form.querySelectorAll('input[type="hidden"][name]')) {
|
|
34
|
+
next.searchParams.set(input.getAttribute('name'), input.getAttribute('value') ?? '');
|
|
35
|
+
}
|
|
36
|
+
if (!urls.owns(next.href))
|
|
37
|
+
throw new GoogleDriveError(`${GoogleDriveFailure.Unrecognized}:foreign-form`);
|
|
38
|
+
return next.href;
|
|
39
|
+
}
|
|
40
|
+
const legacy = document.querySelector('a[href*="export=download"][href*="confirm="]')?.getAttribute('href');
|
|
41
|
+
if (legacy != null) {
|
|
42
|
+
const next = new URL(legacy, base);
|
|
43
|
+
return urls.owns(next.href) ? next.href : undefined;
|
|
44
|
+
}
|
|
45
|
+
return undefined;
|
|
46
|
+
};
|
|
47
|
+
const file = async (id, resourceKey) => {
|
|
48
|
+
let url = urls.downloadUrl(id, resourceKey);
|
|
49
|
+
for (let step = 0; step < DRIVE_PROBE_STEPS; step++) {
|
|
50
|
+
const response = await http.get(url, { headers: { range: 'bytes=0-0' }, maxBytes: DRIVE_PAGE_BYTES });
|
|
51
|
+
const refused = await http.refusal(response);
|
|
52
|
+
if (refused != null)
|
|
53
|
+
throw refused;
|
|
54
|
+
if (isAttachment(response)) {
|
|
55
|
+
await http.discard(response);
|
|
56
|
+
return described(url, response);
|
|
57
|
+
}
|
|
58
|
+
const page = await http.text(response);
|
|
59
|
+
const next = followUp(page, response.url !== '' ? response.url : url);
|
|
60
|
+
if (next == null)
|
|
61
|
+
throw http.pageFailure(page) ?? new GoogleDriveError(GoogleDriveFailure.Unrecognized);
|
|
62
|
+
url = next;
|
|
63
|
+
}
|
|
64
|
+
throw new GoogleDriveError(`${GoogleDriveFailure.Unrecognized}:steps`);
|
|
65
|
+
};
|
|
66
|
+
const exported = async (url) => {
|
|
67
|
+
const response = await http.get(url, { maxBytes: DRIVE_PAGE_BYTES });
|
|
68
|
+
const refused = await http.refusal(response);
|
|
69
|
+
if (refused != null)
|
|
70
|
+
throw refused;
|
|
71
|
+
if (isAttachment(response)) {
|
|
72
|
+
await http.discard(response);
|
|
73
|
+
// An export is generated on the fly: its length (if any) is not worth admitting it by.
|
|
74
|
+
return described(url, response, false);
|
|
75
|
+
}
|
|
76
|
+
throw http.pageFailure(await http.text(response)) ?? new GoogleDriveError(GoogleDriveFailure.Unrecognized);
|
|
77
|
+
};
|
|
78
|
+
return { file, exported };
|
|
79
|
+
};
|
|
80
|
+
//# sourceMappingURL=probe.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"probe.js","sourceRoot":"","sources":["../../src/source/probe.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,SAAS,EAAE,MAAM,UAAU,CAAA;AACpC,OAAO,EAAE,kBAAkB,EAAE,MAAM,cAAc,CAAA;AACjD,OAAO,EAAE,gBAAgB,EAAE,MAAM,cAAc,CAAA;AAE/C,OAAO,EAAE,SAAS,EAAE,MAAM,mBAAmB,CAAA;AAC7C,OAAO,EAAE,gBAAgB,EAAE,iBAAiB,EAAE,MAAM,mBAAmB,CAAA;AAGvE;;;;;GAKG;AACH,MAAM,CAAC,MAAM,cAAc,GAAG,CAAC,IAAe,EAAE,IAA0B,EAAc,EAAE;IACxF,MAAM,YAAY,GAAG,CAAC,QAAkB,EAAW,EAAE,CACnD,aAAa,CAAC,IAAI,CAAC,QAAQ,CAAC,OAAO,CAAC,GAAG,CAAC,qBAAqB,CAAC,IAAI,EAAE,CAAC,IAAI,CAAC,aAAa,CAAC,IAAI,CAAC,QAAQ,CAAC,OAAO,CAAC,GAAG,CAAC,cAAc,CAAC,IAAI,EAAE,CAAC,CAAA;IAE1I,MAAM,MAAM,GAAG,CAAC,QAAkB,EAAsB,EAAE;QACxD,MAAM,KAAK,GAAG,MAAM,CAAC,aAAa,CAAC,IAAI,CAAC,QAAQ,CAAC,OAAO,CAAC,GAAG,CAAC,eAAe,CAAC,IAAI,EAAE,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,GAAG,CAAC,CAAA;QACjG,IAAI,MAAM,CAAC,QAAQ,CAAC,KAAK,CAAC;YAAE,OAAO,KAAK,CAAA;QACxC,MAAM,MAAM,GAAG,MAAM,CAAC,QAAQ,CAAC,OAAO,CAAC,GAAG,CAAC,gBAAgB,CAAC,IAAI,GAAG,CAAC,CAAA;QACpE,OAAO,QAAQ,CAAC,MAAM,KAAK,GAAG,IAAI,MAAM,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,SAAS,CAAA;IAChF,CAAC,CAAA;IAED,MAAM,SAAS,GAAG,CAAC,GAAW,EAAE,QAAkB,EAAE,IAAI,GAAG,IAAI,EAAc,EAAE;QAC7E,MAAM,IAAI,GAAG,SAAS,CAAC,eAAe,CAAC,QAAQ,CAAC,OAAO,CAAC,GAAG,CAAC,qBAAqB,CAAC,CAAC,CAAA;QACnF,MAAM,KAAK,GAAG,IAAI,CAAC,CAAC,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,SAAS,CAAA;QACjD,OAAO,EAAE,GAAG,EAAE,GAAG,CAAC,IAAI,IAAI,IAAI,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,GAAG,CAAC,KAAK,IAAI,IAAI,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,KAAK,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,CAAA;IAC9F,CAAC,CAAA;IAED,wHAAwH;IACxH,MAAM,QAAQ,GAAG,CAAC,IAAY,EAAE,IAAY,EAAsB,EAAE;QAClE,MAAM,EAAE,QAAQ,EAAE,GAAG,SAAS,CAAC,IAAI,CAAC,CAAA;QACpC,MAAM,IAAI,GAAG,QAAQ,CAAC,aAAa,CAAC,oBAAoB,CAAC,IAAI,QAAQ,CAAC,aAAa,CAAC,0BAA0B,CAAC,CAAA;QAC/G,MAAM,MAAM,GAAG,IAAI,EAAE,YAAY,CAAC,QAAQ,CAAC,CAAA;QAC3C,IAAI,IAAI,IAAI,IAAI,IAAI,MAAM,IAAI,IAAI,IAAI,MAAM,KAAK,EAAE,EAAE,CAAC;YACpD,MAAM,IAAI,GAAG,IAAI,GAAG,CAAC,MAAM,EAAE,IAAI,CAAC,CAAA;YAClC,KAAK,MAAM,KAAK,IAAI,IAAI,CAAC,gBAAgB,CAAC,4BAA4B,CAAC,EAAE,CAAC;gBACxE,IAAI,CAAC,YAAY,CAAC,GAAG,CAAC,KAAK,CAAC,YAAY,CAAC,MAAM,CAAE,EAAE,KAAK,CAAC,YAAY,CAAC,OAAO,CAAC,IAAI,EAAE,CAAC,CAAA;YACvF,CAAC;YACD,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC;gBAAE,MAAM,IAAI,gBAAgB,CAAC,GAAG,kBAAkB,CAAC,YAAY,eAAe,CAAC,CAAA;YACxG,OAAO,IAAI,CAAC,IAAI,CAAA;QAClB,CAAC;QACD,MAAM,MAAM,GAAG,QAAQ,CAAC,aAAa,CAAC,8CAA8C,CAAC,EAAE,YAAY,CAAC,MAAM,CAAC,CAAA;QAC3G,IAAI,MAAM,IAAI,IAAI,EAAE,CAAC;YACnB,MAAM,IAAI,GAAG,IAAI,GAAG,CAAC,MAAM,EAAE,IAAI,CAAC,CAAA;YAClC,OAAO,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,SAAS,CAAA;QACrD,CAAC;QACD,OAAO,SAAS,CAAA;IAClB,CAAC,CAAA;IAED,MAAM,IAAI,GAAuB,KAAK,EAAE,EAAE,EAAE,WAAW,EAAE,EAAE;QACzD,IAAI,GAAG,GAAG,IAAI,CAAC,WAAW,CAAC,EAAE,EAAE,WAAW,CAAC,CAAA;QAC3C,KAAK,IAAI,IAAI,GAAG,CAAC,EAAE,IAAI,GAAG,iBAAiB,EAAE,IAAI,EAAE,EAAE,CAAC;YACpD,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,GAAG,CAAC,GAAG,EAAE,EAAE,OAAO,EAAE,EAAE,KAAK,EAAE,WAAW,EAAE,EAAE,QAAQ,EAAE,gBAAgB,EAAE,CAAC,CAAA;YACrG,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAA;YAC5C,IAAI,OAAO,IAAI,IAAI;gBAAE,MAAM,OAAO,CAAA;YAClC,IAAI,YAAY,CAAC,QAAQ,CAAC,EAAE,CAAC;gBAC3B,MAAM,IAAI,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAA;gBAC5B,OAAO,SAAS,CAAC,GAAG,EAAE,QAAQ,CAAC,CAAA;YACjC,CAAC;YACD,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAA;YACtC,MAAM,IAAI,GAAG,QAAQ,CAAC,IAAI,EAAE,QAAQ,CAAC,GAAG,KAAK,EAAE,CAAC,CAAC,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC,CAAA;YACrE,IAAI,IAAI,IAAI,IAAI;gBAAE,MAAM,IAAI,CAAC,WAAW,CAAC,IAAI,CAAC,IAAI,IAAI,gBAAgB,CAAC,kBAAkB,CAAC,YAAY,CAAC,CAAA;YACvG,GAAG,GAAG,IAAI,CAAA;QACZ,CAAC;QACD,MAAM,IAAI,gBAAgB,CAAC,GAAG,kBAAkB,CAAC,YAAY,QAAQ,CAAC,CAAA;IACxE,CAAC,CAAA;IAED,MAAM,QAAQ,GAA2B,KAAK,EAAC,GAAG,EAAC,EAAE;QACnD,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,GAAG,CAAC,GAAG,EAAE,EAAE,QAAQ,EAAE,gBAAgB,EAAE,CAAC,CAAA;QACpE,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAA;QAC5C,IAAI,OAAO,IAAI,IAAI;YAAE,MAAM,OAAO,CAAA;QAClC,IAAI,YAAY,CAAC,QAAQ,CAAC,EAAE,CAAC;YAC3B,MAAM,IAAI,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAA;YAC5B,uFAAuF;YACvF,OAAO,SAAS,CAAC,GAAG,EAAE,QAAQ,EAAE,KAAK,CAAC,CAAA;QACxC,CAAC;QACD,MAAM,IAAI,CAAC,WAAW,CAAC,MAAM,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC,IAAI,IAAI,gBAAgB,CAAC,kBAAkB,CAAC,YAAY,CAAC,CAAA;IAC5G,CAAC,CAAA;IAED,OAAO,EAAE,IAAI,EAAE,QAAQ,EAAE,CAAA;AAC3B,CAAC,CAAA"}
|
|
@@ -0,0 +1,87 @@
|
|
|
1
|
+
import type { SourceFetchInit } from '@owlmeans/llm-reader';
|
|
2
|
+
import type { GoogleDriveKind } from '../consts.js';
|
|
3
|
+
import type { GoogleDriveError } from '../errors.js';
|
|
4
|
+
import type { GoogleDriveLink } from '../types.js';
|
|
5
|
+
/** One folder to list, where its files go and how deep it is. */
|
|
6
|
+
export interface DriveFolder {
|
|
7
|
+
id: string;
|
|
8
|
+
resourceKey?: string;
|
|
9
|
+
/** The folder's path inside the linked folder, `''` or ending in `/`. */
|
|
10
|
+
prefix: string;
|
|
11
|
+
/** `0` for the linked folder. */
|
|
12
|
+
level: number;
|
|
13
|
+
}
|
|
14
|
+
/** A folder's child as a listing names it. */
|
|
15
|
+
export interface DriveItem {
|
|
16
|
+
id: string;
|
|
17
|
+
kind: GoogleDriveKind;
|
|
18
|
+
name: string;
|
|
19
|
+
/** Its real media type — never a Google item type. */
|
|
20
|
+
mime?: string;
|
|
21
|
+
size?: number;
|
|
22
|
+
resourceKey?: string;
|
|
23
|
+
}
|
|
24
|
+
/** A link with what Drive told about it. */
|
|
25
|
+
export interface ResolvedLink extends GoogleDriveLink {
|
|
26
|
+
name?: string;
|
|
27
|
+
mime?: string;
|
|
28
|
+
size?: number;
|
|
29
|
+
}
|
|
30
|
+
/** A file's bytes: the URL a plain GET downloads, with what Drive declared. */
|
|
31
|
+
export interface ProbedFile {
|
|
32
|
+
url: string;
|
|
33
|
+
name?: string;
|
|
34
|
+
size?: number;
|
|
35
|
+
}
|
|
36
|
+
/** One way of listing Drive folders — the keyless embedded view, or Drive API v3. */
|
|
37
|
+
export interface DriveLister {
|
|
38
|
+
/** A folder's children, up to about `limit`. Throws `GoogleDriveError` when Drive refuses. */
|
|
39
|
+
list: (folder: DriveFolder, limit: number) => Promise<DriveItem[]>;
|
|
40
|
+
/** What an `open?id=` link points at. */
|
|
41
|
+
resolve: (url: string, link: GoogleDriveLink) => Promise<ResolvedLink>;
|
|
42
|
+
}
|
|
43
|
+
/** Downloads asked before they are handed to the collector. */
|
|
44
|
+
export interface DriveProbe {
|
|
45
|
+
/**
|
|
46
|
+
* A stored file's download followed through Drive's warning form: a URL that serves the bytes to a
|
|
47
|
+
* plain GET, its name and size. Throws `GoogleDriveError` for a private, missing or quota-bound file.
|
|
48
|
+
*/
|
|
49
|
+
file: (id: string, resourceKey?: string) => Promise<ProbedFile>;
|
|
50
|
+
/** An editor file's export, asked for its name (its body is dropped). */
|
|
51
|
+
exported: (url: string) => Promise<ProbedFile>;
|
|
52
|
+
}
|
|
53
|
+
/** Requests to Google and the reading of its refusals. */
|
|
54
|
+
export interface DriveHttp {
|
|
55
|
+
/** A GET through the reader's guarded fetch; a file-streams Drive resolver's refusal becomes `GoogleDriveError('resolver')`. */
|
|
56
|
+
get: (url: string, init?: SourceFetchInit) => Promise<Response>;
|
|
57
|
+
/**
|
|
58
|
+
* The refusal an answer stands for — Google's sign-in host, 401/403/404/429, quota and API-key
|
|
59
|
+
* errors, any other status ≥ 400 — or `undefined` for a usable answer. Reads the body of a refusal only.
|
|
60
|
+
*/
|
|
61
|
+
refusal: (response: Response) => Promise<GoogleDriveError | undefined>;
|
|
62
|
+
/** What an HTML page served instead of the expected answer says: quota, sign-in, no access. */
|
|
63
|
+
pageFailure: (page: string) => GoogleDriveError | undefined;
|
|
64
|
+
/** A body read as text; `''` when it cannot be read. */
|
|
65
|
+
text: (response: Response) => Promise<string>;
|
|
66
|
+
/** Drop a body that is not read. */
|
|
67
|
+
discard: (response: Response) => Promise<void>;
|
|
68
|
+
}
|
|
69
|
+
/** A Drive API v3 file resource — the fields asked for. */
|
|
70
|
+
export interface DriveApiFile {
|
|
71
|
+
id?: string;
|
|
72
|
+
name?: string;
|
|
73
|
+
mimeType?: string;
|
|
74
|
+
size?: string;
|
|
75
|
+
resourceKey?: string;
|
|
76
|
+
shortcutDetails?: {
|
|
77
|
+
targetId?: string;
|
|
78
|
+
targetMimeType?: string;
|
|
79
|
+
targetResourceKey?: string;
|
|
80
|
+
};
|
|
81
|
+
}
|
|
82
|
+
/** A Drive API v3 `files.list` page. */
|
|
83
|
+
export interface DriveApiList {
|
|
84
|
+
nextPageToken?: string;
|
|
85
|
+
files?: DriveApiFile[];
|
|
86
|
+
}
|
|
87
|
+
//# sourceMappingURL=types.local.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"types.local.d.ts","sourceRoot":"","sources":["../../src/source/types.local.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,sBAAsB,CAAA;AAC3D,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,cAAc,CAAA;AACnD,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,cAAc,CAAA;AACpD,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,aAAa,CAAA;AAElD,iEAAiE;AACjE,MAAM,WAAW,WAAW;IAC1B,EAAE,EAAE,MAAM,CAAA;IACV,WAAW,CAAC,EAAE,MAAM,CAAA;IACpB,yEAAyE;IACzE,MAAM,EAAE,MAAM,CAAA;IACd,iCAAiC;IACjC,KAAK,EAAE,MAAM,CAAA;CACd;AAED,8CAA8C;AAC9C,MAAM,WAAW,SAAS;IACxB,EAAE,EAAE,MAAM,CAAA;IACV,IAAI,EAAE,eAAe,CAAA;IACrB,IAAI,EAAE,MAAM,CAAA;IACZ,sDAAsD;IACtD,IAAI,CAAC,EAAE,MAAM,CAAA;IACb,IAAI,CAAC,EAAE,MAAM,CAAA;IACb,WAAW,CAAC,EAAE,MAAM,CAAA;CACrB;AAED,4CAA4C;AAC5C,MAAM,WAAW,YAAa,SAAQ,eAAe;IACnD,IAAI,CAAC,EAAE,MAAM,CAAA;IACb,IAAI,CAAC,EAAE,MAAM,CAAA;IACb,IAAI,CAAC,EAAE,MAAM,CAAA;CACd;AAED,+EAA+E;AAC/E,MAAM,WAAW,UAAU;IACzB,GAAG,EAAE,MAAM,CAAA;IACX,IAAI,CAAC,EAAE,MAAM,CAAA;IACb,IAAI,CAAC,EAAE,MAAM,CAAA;CACd;AAED,qFAAqF;AACrF,MAAM,WAAW,WAAW;IAC1B,8FAA8F;IAC9F,IAAI,EAAE,CAAC,MAAM,EAAE,WAAW,EAAE,KAAK,EAAE,MAAM,KAAK,OAAO,CAAC,SAAS,EAAE,CAAC,CAAA;IAClE,yCAAyC;IACzC,OAAO,EAAE,CAAC,GAAG,EAAE,MAAM,EAAE,IAAI,EAAE,eAAe,KAAK,OAAO,CAAC,YAAY,CAAC,CAAA;CACvE;AAED,+DAA+D;AAC/D,MAAM,WAAW,UAAU;IACzB;;;OAGG;IACH,IAAI,EAAE,CAAC,EAAE,EAAE,MAAM,EAAE,WAAW,CAAC,EAAE,MAAM,KAAK,OAAO,CAAC,UAAU,CAAC,CAAA;IAC/D,yEAAyE;IACzE,QAAQ,EAAE,CAAC,GAAG,EAAE,MAAM,KAAK,OAAO,CAAC,UAAU,CAAC,CAAA;CAC/C;AAED,0DAA0D;AAC1D,MAAM,WAAW,SAAS;IACxB,gIAAgI;IAChI,GAAG,EAAE,CAAC,GAAG,EAAE,MAAM,EAAE,IAAI,CAAC,EAAE,eAAe,KAAK,OAAO,CAAC,QAAQ,CAAC,CAAA;IAC/D;;;OAGG;IACH,OAAO,EAAE,CAAC,QAAQ,EAAE,QAAQ,KAAK,OAAO,CAAC,gBAAgB,GAAG,SAAS,CAAC,CAAA;IACtE,+FAA+F;IAC/F,WAAW,EAAE,CAAC,IAAI,EAAE,MAAM,KAAK,gBAAgB,GAAG,SAAS,CAAA;IAC3D,wDAAwD;IACxD,IAAI,EAAE,CAAC,QAAQ,EAAE,QAAQ,KAAK,OAAO,CAAC,MAAM,CAAC,CAAA;IAC7C,oCAAoC;IACpC,OAAO,EAAE,CAAC,QAAQ,EAAE,QAAQ,KAAK,OAAO,CAAC,IAAI,CAAC,CAAA;CAC/C;AAED,2DAA2D;AAC3D,MAAM,WAAW,YAAY;IAC3B,EAAE,CAAC,EAAE,MAAM,CAAA;IACX,IAAI,CAAC,EAAE,MAAM,CAAA;IACb,QAAQ,CAAC,EAAE,MAAM,CAAA;IACjB,IAAI,CAAC,EAAE,MAAM,CAAA;IACb,WAAW,CAAC,EAAE,MAAM,CAAA;IACpB,eAAe,CAAC,EAAE;QAAE,QAAQ,CAAC,EAAE,MAAM,CAAC;QAAC,cAAc,CAAC,EAAE,MAAM,CAAC;QAAC,iBAAiB,CAAC,EAAE,MAAM,CAAA;KAAE,CAAA;CAC7F;AAED,wCAAwC;AACxC,MAAM,WAAW,YAAY;IAC3B,aAAa,CAAC,EAAE,MAAM,CAAA;IACtB,KAAK,CAAC,EAAE,YAAY,EAAE,CAAA;CACvB"}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"types.local.js","sourceRoot":"","sources":["../../src/source/types.local.ts"],"names":[],"mappings":""}
|
|
@@ -0,0 +1,22 @@
|
|
|
1
|
+
import type { ReaderSource } from '@owlmeans/llm-reader';
|
|
2
|
+
import type { GoogleDriveSourceOptions } from './types.js';
|
|
3
|
+
/**
|
|
4
|
+
* Google Drive as a reader source — public folder, file and Docs links expanded into entries the
|
|
5
|
+
* collector downloads with a plain GET:
|
|
6
|
+
*
|
|
7
|
+
* - a folder (`drive/folders/<id>`, `open?id=` of a folder) is listed breadth-first down to `depth`
|
|
8
|
+
* sub-folder levels, at most `maxEntries` entries (and the collect's `maxFiles + 1`, so the cut is
|
|
9
|
+
* reported) from at most `maxFolders` folders — keyless from the embedded folder view, or through
|
|
10
|
+
* Drive API v3 with `apiKey`; entry paths are `<sub-folder>/<name>` relative to the linked folder;
|
|
11
|
+
* - a stored file becomes its usercontent download (`confirm=t`); a file whose size is unknown or
|
|
12
|
+
* past Drive's scan limit is probed first (one ranged GET) for its name, size and — behind Drive's
|
|
13
|
+
* warning form — the URL that really serves the bytes; a file Drive refuses is left out and logged;
|
|
14
|
+
* - an editor file (Docs, Sheets, Slides, Drawings) becomes its export in the configured format, its
|
|
15
|
+
* name with that extension;
|
|
16
|
+
* - a single file or editor link is one entry named as Drive names it (`Content-Disposition`).
|
|
17
|
+
*
|
|
18
|
+
* A private item throws `GoogleDriveError` (`not-public`), an exhausted quota `quota`, a 404
|
|
19
|
+
* `not-found`. Every request is the reader's guarded fetch, under its timeout and the collect's signal.
|
|
20
|
+
*/
|
|
21
|
+
export declare const createGoogleDriveSource: (options?: GoogleDriveSourceOptions) => ReaderSource;
|
|
22
|
+
//# sourceMappingURL=source.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"source.d.ts","sourceRoot":"","sources":["../src/source.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,YAAY,EAAoC,MAAM,sBAAsB,CAAA;AAa1F,OAAO,KAAK,EAAE,wBAAwB,EAAoB,MAAM,YAAY,CAAA;AAoB5E;;;;;;;;;;;;;;;;;GAiBG;AACH,eAAO,MAAM,uBAAuB,aAAa,wBAAwB,KAAQ,YA+HhF,CAAA"}
|