@nitpicker/crawler 0.6.2 → 0.6.5-alpha.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/lib/archive/database.d.ts +1 -30
- package/lib/archive/database.js +6 -187
- package/lib/archive/get-json.d.ts +10 -0
- package/lib/archive/get-json.js +25 -0
- package/lib/archive/init-schema.d.ts +8 -0
- package/lib/archive/init-schema.js +119 -0
- package/lib/archive/limited-page-ids.d.ts +8 -0
- package/lib/archive/limited-page-ids.js +17 -0
- package/lib/archive/page.d.ts +1 -48
- package/lib/archive/redirect-table.d.ts +7 -0
- package/lib/archive/redirect-table.js +23 -0
- package/lib/archive/types.d.ts +76 -0
- package/lib/crawler.d.ts +1 -1
- package/package.json +2 -2
- package/lib/archive/filesystem/index.d.ts +0 -17
- package/lib/archive/filesystem/index.js +0 -17
- package/lib/archive/filesystem/utils.d.ts +0 -109
- package/lib/archive/filesystem/utils.js +0 -185
- package/lib/archive/filesystem/zip.d.ts +0 -29
- package/lib/archive/filesystem/zip.js +0 -53
- package/lib/archive/index.d.ts +0 -6
- package/lib/archive/index.js +0 -11
- package/lib/crawler/index.d.ts +0 -2
- package/lib/crawler/index.js +0 -2
- package/lib/crawler/network.d.ts +0 -30
- package/lib/crawler/network.js +0 -226
- package/lib/crawler/result-handler.d.ts +0 -118
- package/lib/crawler/result-handler.js +0 -153
- package/lib/crawler/speculative-pagination.d.ts +0 -52
- package/lib/crawler/speculative-pagination.js +0 -215
- package/lib/crawler/url-filter.d.ts +0 -56
- package/lib/crawler/url-filter.js +0 -110
- package/lib/index.d.ts +0 -16
- package/lib/index.js +0 -18
- package/lib/qzilla.d.ts +0 -136
- package/lib/qzilla.js +0 -292
- package/lib/utils/array/index.d.ts +0 -1
- package/lib/utils/array/index.js +0 -1
- package/lib/utils/async/index.d.ts +0 -1
- package/lib/utils/async/index.js +0 -1
- package/lib/utils/error/index.d.ts +0 -3
- package/lib/utils/error/index.js +0 -2
- package/lib/utils/event-emitter/index.d.ts +0 -6
- package/lib/utils/event-emitter/index.js +0 -6
- package/lib/utils/index.d.ts +0 -5
- package/lib/utils/index.js +0 -5
- package/lib/utils/network/index.d.ts +0 -1
- package/lib/utils/network/index.js +0 -1
- package/lib/utils/object/index.d.ts +0 -1
- package/lib/utils/object/index.js +0 -1
- package/lib/utils/path/index.d.ts +0 -1
- package/lib/utils/path/index.js +0 -1
- package/lib/utils/path/safe-filepath.d.ts +0 -7
- package/lib/utils/path/safe-filepath.js +0 -12
- package/lib/utils/regexp/index.d.ts +0 -1
- package/lib/utils/regexp/index.js +0 -1
- package/lib/utils/retryable/index.d.ts +0 -2
- package/lib/utils/retryable/index.js +0 -1
- package/lib/utils/sort/index.d.ts +0 -14
- package/lib/utils/sort/index.js +0 -61
- package/lib/utils/sort/remove-matches.d.ts +0 -9
- package/lib/utils/sort/remove-matches.js +0 -23
- package/lib/utils/types/index.d.ts +0 -1
- package/lib/utils/types/index.js +0 -1
- package/lib/utils/url/index.d.ts +0 -5
- package/lib/utils/url/index.js +0 -5
- package/lib/utils/url/is-lower-layer.d.ts +0 -15
- package/lib/utils/url/is-lower-layer.js +0 -55
- package/lib/utils/url/parse-url.d.ts +0 -11
- package/lib/utils/url/parse-url.js +0 -20
- package/lib/utils/url/path-match.d.ts +0 -11
- package/lib/utils/url/path-match.js +0 -18
- package/lib/utils/url/sort-url.d.ts +0 -10
- package/lib/utils/url/sort-url.js +0 -24
- package/lib/utils/url/url-partial-match.d.ts +0 -11
- package/lib/utils/url/url-partial-match.js +0 -32
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import type { Config, DB_Anchor, DB_Page, DB_Redirect, DB_Referrer, DB_Resource, DatabaseEvent, PageFilter } from './types.js';
|
|
1
|
+
import type { Config, DatabaseOption, DB_Anchor, DB_Page, DB_Redirect, DB_Referrer, DB_Resource, DatabaseEvent, PageFilter } from './types.js';
|
|
2
2
|
import type { PageData, Resource } from '../utils/types/types.js';
|
|
3
3
|
import type { Knex } from 'knex';
|
|
4
4
|
import { TypedAwaitEventEmitter as EventEmitter } from '@d-zero/shared/typed-await-event-emitter';
|
|
@@ -199,32 +199,3 @@ export declare class Database extends EventEmitter<DatabaseEvent> {
|
|
|
199
199
|
*/
|
|
200
200
|
static connect(options: DatabaseOption): Promise<Database>;
|
|
201
201
|
}
|
|
202
|
-
/**
|
|
203
|
-
* Base options shared by all database connection configurations.
|
|
204
|
-
*/
|
|
205
|
-
type AbsDatabaseOption = {
|
|
206
|
-
/** The working directory for the database (used for resolving relative paths). */
|
|
207
|
-
workingDir: string;
|
|
208
|
-
};
|
|
209
|
-
/**
|
|
210
|
-
* Union type for all supported database connection options.
|
|
211
|
-
*/
|
|
212
|
-
type DatabaseOption = DatabaseSqlite3Option | DatabaseMySqlOption;
|
|
213
|
-
/**
|
|
214
|
-
* Connection options for a SQLite3 database.
|
|
215
|
-
*/
|
|
216
|
-
type DatabaseSqlite3Option = AbsDatabaseOption & {
|
|
217
|
-
/** The database type identifier. */
|
|
218
|
-
type: 'sqlite3';
|
|
219
|
-
/** The absolute file path to the SQLite database file. */
|
|
220
|
-
filename: string;
|
|
221
|
-
};
|
|
222
|
-
/**
|
|
223
|
-
* Connection options for a MySQL database.
|
|
224
|
-
* Note: MySQL support is not yet implemented.
|
|
225
|
-
*/
|
|
226
|
-
type DatabaseMySqlOption = AbsDatabaseOption & {
|
|
227
|
-
/** The database type identifier. */
|
|
228
|
-
type: 'mysql';
|
|
229
|
-
};
|
|
230
|
-
export {};
|
package/lib/archive/database.js
CHANGED
|
@@ -42,6 +42,10 @@ import { eachSplitted } from '../utils/array/each-splitted.js';
|
|
|
42
42
|
import { ErrorEmitter } from '../utils/error/error-emitter.js';
|
|
43
43
|
import { dbLog } from './debug.js';
|
|
44
44
|
import { mkdir } from './filesystem/mkdir.js';
|
|
45
|
+
import { getJSON } from './get-json.js';
|
|
46
|
+
import { initSchema } from './init-schema.js';
|
|
47
|
+
import { limitedPageIds } from './limited-page-ids.js';
|
|
48
|
+
import { redirectTable } from './redirect-table.js';
|
|
45
49
|
const retrySetting = {
|
|
46
50
|
interval: 300,
|
|
47
51
|
retries: 3,
|
|
@@ -747,121 +751,10 @@ let Database = (() => {
|
|
|
747
751
|
}
|
|
748
752
|
/**
|
|
749
753
|
* Initializes the database schema if tables do not exist.
|
|
750
|
-
*
|
|
751
|
-
* (`info`, `pages`, `anchors`, `images`, `resources`, `resources-referrers`).
|
|
754
|
+
* Delegates to {@link initSchema} for the actual table creation.
|
|
752
755
|
*/
|
|
753
756
|
async #init() {
|
|
754
|
-
|
|
755
|
-
if (isExists) {
|
|
756
|
-
return;
|
|
757
|
-
}
|
|
758
|
-
// Enable WAL mode and foreign keys for better performance and data integrity
|
|
759
|
-
await this.#instance.raw('PRAGMA journal_mode = WAL');
|
|
760
|
-
await this.#instance.raw('PRAGMA foreign_keys = ON');
|
|
761
|
-
await this.#instance.schema
|
|
762
|
-
.createTable('info', (t) => {
|
|
763
|
-
t.increments('id');
|
|
764
|
-
t.string('version');
|
|
765
|
-
t.string('name');
|
|
766
|
-
t.string('baseUrl');
|
|
767
|
-
t.boolean('recursive');
|
|
768
|
-
t.integer('interval');
|
|
769
|
-
t.boolean('image');
|
|
770
|
-
t.boolean('fetchExternal');
|
|
771
|
-
t.integer('parallels');
|
|
772
|
-
t.json('scope');
|
|
773
|
-
t.json('excludes');
|
|
774
|
-
t.json('excludeKeywords');
|
|
775
|
-
t.json('excludeUrls');
|
|
776
|
-
t.integer('maxExcludedDepth');
|
|
777
|
-
t.integer('retry');
|
|
778
|
-
t.boolean('fromList');
|
|
779
|
-
t.boolean('disableQueries');
|
|
780
|
-
t.string('userAgent');
|
|
781
|
-
t.boolean('ignoreRobots');
|
|
782
|
-
})
|
|
783
|
-
.createTable('pages', (t) => {
|
|
784
|
-
t.increments('id');
|
|
785
|
-
t.string('url', 8190).notNullable().unique();
|
|
786
|
-
t.integer('redirectDestId').unsigned().references('pages.id').defaultTo(null);
|
|
787
|
-
t.boolean('scraped').notNullable();
|
|
788
|
-
t.boolean('isTarget').notNullable();
|
|
789
|
-
t.boolean('isExternal');
|
|
790
|
-
t.integer('status');
|
|
791
|
-
t.string('statusText');
|
|
792
|
-
t.string('contentType').nullable();
|
|
793
|
-
t.integer('contentLength').unsigned().nullable();
|
|
794
|
-
t.json('responseHeaders').nullable();
|
|
795
|
-
t.string('lang');
|
|
796
|
-
t.string('title');
|
|
797
|
-
t.string('description');
|
|
798
|
-
t.string('keywords');
|
|
799
|
-
t.boolean('noindex');
|
|
800
|
-
t.boolean('nofollow');
|
|
801
|
-
t.boolean('noarchive');
|
|
802
|
-
t.string('canonical');
|
|
803
|
-
t.string('alternate');
|
|
804
|
-
t.string('og_type');
|
|
805
|
-
t.string('og_title');
|
|
806
|
-
t.string('og_site_name');
|
|
807
|
-
t.string('og_description');
|
|
808
|
-
t.string('og_url');
|
|
809
|
-
t.string('og_image');
|
|
810
|
-
t.string('twitter_card');
|
|
811
|
-
t.string('html');
|
|
812
|
-
t.boolean('isSkipped');
|
|
813
|
-
t.string('skipReason');
|
|
814
|
-
t.integer('order').unsigned().nullable();
|
|
815
|
-
t.index('isExternal');
|
|
816
|
-
t.index('contentType');
|
|
817
|
-
t.index('scraped');
|
|
818
|
-
t.index('redirectDestId');
|
|
819
|
-
t.index('order');
|
|
820
|
-
})
|
|
821
|
-
.createTable('anchors', (t) => {
|
|
822
|
-
t.increments('id');
|
|
823
|
-
t.integer('pageId').notNullable().unsigned().references('pages.id');
|
|
824
|
-
t.integer('hrefId').notNullable().unsigned().references('pages.id');
|
|
825
|
-
t.string('hash');
|
|
826
|
-
t.string('textContent').nullable();
|
|
827
|
-
t.index('pageId');
|
|
828
|
-
t.index('hrefId');
|
|
829
|
-
})
|
|
830
|
-
.createTable('images', (t) => {
|
|
831
|
-
t.increments('id');
|
|
832
|
-
t.integer('pageId').notNullable().unsigned().references('pages.id');
|
|
833
|
-
t.string('src', 8190);
|
|
834
|
-
t.string('currentSrc', 8190);
|
|
835
|
-
t.string('alt');
|
|
836
|
-
t.float('width').unsigned().notNullable();
|
|
837
|
-
t.float('height').unsigned().notNullable();
|
|
838
|
-
t.integer('naturalWidth').unsigned().notNullable();
|
|
839
|
-
t.integer('naturalHeight').unsigned().notNullable();
|
|
840
|
-
t.boolean('isLazy');
|
|
841
|
-
t.integer('viewportWidth').unsigned().notNullable();
|
|
842
|
-
t.string('sourceCode');
|
|
843
|
-
t.index('pageId');
|
|
844
|
-
})
|
|
845
|
-
.createTable('resources', (t) => {
|
|
846
|
-
t.increments('id');
|
|
847
|
-
t.string('url', 8190).notNullable().unique();
|
|
848
|
-
t.boolean('isExternal');
|
|
849
|
-
t.integer('status');
|
|
850
|
-
t.string('statusText');
|
|
851
|
-
t.string('contentType').nullable();
|
|
852
|
-
t.integer('contentLength').unsigned().nullable();
|
|
853
|
-
t.string('compress').nullable();
|
|
854
|
-
t.string('cdn').nullable();
|
|
855
|
-
t.json('responseHeaders').nullable();
|
|
856
|
-
})
|
|
857
|
-
.createTable('resources-referrers', (t) => {
|
|
858
|
-
t.increments('id');
|
|
859
|
-
t.integer('resourceId').notNullable().unsigned().references('resources.id');
|
|
860
|
-
t.integer('pageId').notNullable().unsigned().references('pages.id');
|
|
861
|
-
t.unique(['resourceId', 'pageId']);
|
|
862
|
-
t.index('resourceId');
|
|
863
|
-
t.index('pageId');
|
|
864
|
-
});
|
|
757
|
+
await initSchema(this.#instance);
|
|
865
758
|
}
|
|
866
759
|
/**
|
|
867
760
|
* Upserts page data into the `pages` table (inserts if new, updates if existing).
|
|
@@ -942,77 +835,3 @@ let Database = (() => {
|
|
|
942
835
|
};
|
|
943
836
|
})();
|
|
944
837
|
export { Database };
|
|
945
|
-
// ----- ----- ----- ----- -----
|
|
946
|
-
//
|
|
947
|
-
// Common Queries
|
|
948
|
-
//
|
|
949
|
-
// ----- ----- ----- ----- -----
|
|
950
|
-
/**
|
|
951
|
-
* Returns a Knex subquery builder that selects page IDs with pagination,
|
|
952
|
-
* ordered by the `order` column (nulls last), excluding redirected pages.
|
|
953
|
-
* @param limit - The maximum number of page IDs to return.
|
|
954
|
-
* @param offset - The number of page IDs to skip before returning results.
|
|
955
|
-
*/
|
|
956
|
-
function limitedPageIds(limit, offset) {
|
|
957
|
-
return async (qb) => {
|
|
958
|
-
await qb
|
|
959
|
-
.select('id')
|
|
960
|
-
.from('pages')
|
|
961
|
-
.orderByRaw('`order` ASC NULLS LAST')
|
|
962
|
-
.whereNull('redirectDestId')
|
|
963
|
-
.limit(limit)
|
|
964
|
-
.offset(offset);
|
|
965
|
-
};
|
|
966
|
-
}
|
|
967
|
-
/**
|
|
968
|
-
* Returns a Knex subquery builder that joins pages with their redirect destinations.
|
|
969
|
-
* When `includeNull` is true, also includes pages without redirects (self-referencing).
|
|
970
|
-
* @param includeNull - Whether to include non-redirected pages in the result. Defaults to `true`.
|
|
971
|
-
*/
|
|
972
|
-
function redirectTable(includeNull = true) {
|
|
973
|
-
return async (qb) => {
|
|
974
|
-
const list = qb
|
|
975
|
-
.select('A.id as fromId', 'A.url as from', 'B.url as to', 'B.id as toId')
|
|
976
|
-
.from('pages as A')
|
|
977
|
-
.join('pages as B', (j) => {
|
|
978
|
-
j.on('A.redirectDestId', '=', 'B.id').andOnNotNull('A.redirectDestId');
|
|
979
|
-
});
|
|
980
|
-
if (includeNull) {
|
|
981
|
-
await list.union(async (qb) => {
|
|
982
|
-
await qb
|
|
983
|
-
.select('A.id as fromId', 'A.url as from', 'A.url as to', 'A.id as toId')
|
|
984
|
-
.from('pages as A')
|
|
985
|
-
.whereNull('A.redirectDestId');
|
|
986
|
-
});
|
|
987
|
-
}
|
|
988
|
-
};
|
|
989
|
-
}
|
|
990
|
-
// ----- ----- ----- ----- -----
|
|
991
|
-
//
|
|
992
|
-
// Utils
|
|
993
|
-
//
|
|
994
|
-
// ----- ----- ----- ----- -----
|
|
995
|
-
/**
|
|
996
|
-
* Safely parses a JSON string, returning a fallback value if parsing fails or the input is not a string.
|
|
997
|
-
* Logs a warning via {@link dbLog} when invalid JSON is detected, including a truncated preview
|
|
998
|
-
* of the data and the parse error message.
|
|
999
|
-
* @template T The expected type of the parsed JSON value and the fallback.
|
|
1000
|
-
* @param data - The data to parse. Only string values are parsed; other types return the fallback.
|
|
1001
|
-
* @param fallback - The value to return if parsing fails or the result is falsy.
|
|
1002
|
-
* @returns The parsed JSON value, or the fallback.
|
|
1003
|
-
*/
|
|
1004
|
-
function getJSON(data, fallback) {
|
|
1005
|
-
try {
|
|
1006
|
-
if (typeof data === 'string') {
|
|
1007
|
-
const result = JSON.parse(data);
|
|
1008
|
-
if (result) {
|
|
1009
|
-
return result;
|
|
1010
|
-
}
|
|
1011
|
-
return fallback;
|
|
1012
|
-
}
|
|
1013
|
-
}
|
|
1014
|
-
catch (error) {
|
|
1015
|
-
dbLog('Warning: Invalid JSON detected in database field. Using fallback value. Data: %s, Error: %s', String(data).slice(0, 200), error instanceof Error ? error.message : String(error));
|
|
1016
|
-
}
|
|
1017
|
-
return fallback;
|
|
1018
|
-
}
|
|
@@ -0,0 +1,10 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Safely parses a JSON string, returning a fallback value if parsing fails or the input is not a string.
|
|
3
|
+
* Logs a warning via {@link dbLog} when invalid JSON is detected, including a truncated preview
|
|
4
|
+
* of the data and the parse error message.
|
|
5
|
+
* @template T The expected type of the parsed JSON value and the fallback.
|
|
6
|
+
* @param data - The data to parse. Only string values are parsed; other types return the fallback.
|
|
7
|
+
* @param fallback - The value to return if parsing fails or the result is falsy.
|
|
8
|
+
* @returns The parsed JSON value, or the fallback.
|
|
9
|
+
*/
|
|
10
|
+
export declare function getJSON<T>(data: unknown, fallback: T): T;
|
|
@@ -0,0 +1,25 @@
|
|
|
1
|
+
import { dbLog } from './debug.js';
|
|
2
|
+
/**
|
|
3
|
+
* Safely parses a JSON string, returning a fallback value if parsing fails or the input is not a string.
|
|
4
|
+
* Logs a warning via {@link dbLog} when invalid JSON is detected, including a truncated preview
|
|
5
|
+
* of the data and the parse error message.
|
|
6
|
+
* @template T The expected type of the parsed JSON value and the fallback.
|
|
7
|
+
* @param data - The data to parse. Only string values are parsed; other types return the fallback.
|
|
8
|
+
* @param fallback - The value to return if parsing fails or the result is falsy.
|
|
9
|
+
* @returns The parsed JSON value, or the fallback.
|
|
10
|
+
*/
|
|
11
|
+
export function getJSON(data, fallback) {
|
|
12
|
+
try {
|
|
13
|
+
if (typeof data === 'string') {
|
|
14
|
+
const result = JSON.parse(data);
|
|
15
|
+
if (result) {
|
|
16
|
+
return result;
|
|
17
|
+
}
|
|
18
|
+
return fallback;
|
|
19
|
+
}
|
|
20
|
+
}
|
|
21
|
+
catch (error) {
|
|
22
|
+
dbLog('Warning: Invalid JSON detected in database field. Using fallback value. Data: %s, Error: %s', String(data).slice(0, 200), error instanceof Error ? error.message : String(error));
|
|
23
|
+
}
|
|
24
|
+
return fallback;
|
|
25
|
+
}
|
|
@@ -0,0 +1,8 @@
|
|
|
1
|
+
import type { Knex } from 'knex';
|
|
2
|
+
/**
|
|
3
|
+
* Initializes the archive database schema if tables do not exist.
|
|
4
|
+
* Enables WAL journal mode and foreign keys, then creates all tables
|
|
5
|
+
* (`info`, `pages`, `anchors`, `images`, `resources`, `resources-referrers`).
|
|
6
|
+
* @param instance - The Knex query builder instance connected to the database.
|
|
7
|
+
*/
|
|
8
|
+
export declare function initSchema(instance: Knex): Promise<void>;
|
|
@@ -0,0 +1,119 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Initializes the archive database schema if tables do not exist.
|
|
3
|
+
* Enables WAL journal mode and foreign keys, then creates all tables
|
|
4
|
+
* (`info`, `pages`, `anchors`, `images`, `resources`, `resources-referrers`).
|
|
5
|
+
* @param instance - The Knex query builder instance connected to the database.
|
|
6
|
+
*/
|
|
7
|
+
export async function initSchema(instance) {
|
|
8
|
+
const isExists = await instance.schema.hasTable('info');
|
|
9
|
+
if (isExists) {
|
|
10
|
+
return;
|
|
11
|
+
}
|
|
12
|
+
// Enable WAL mode and foreign keys for better performance and data integrity
|
|
13
|
+
await instance.raw('PRAGMA journal_mode = WAL');
|
|
14
|
+
await instance.raw('PRAGMA foreign_keys = ON');
|
|
15
|
+
await instance.schema
|
|
16
|
+
.createTable('info', (t) => {
|
|
17
|
+
t.increments('id');
|
|
18
|
+
t.string('version');
|
|
19
|
+
t.string('name');
|
|
20
|
+
t.string('baseUrl');
|
|
21
|
+
t.boolean('recursive');
|
|
22
|
+
t.integer('interval');
|
|
23
|
+
t.boolean('image');
|
|
24
|
+
t.boolean('fetchExternal');
|
|
25
|
+
t.integer('parallels');
|
|
26
|
+
t.json('scope');
|
|
27
|
+
t.json('excludes');
|
|
28
|
+
t.json('excludeKeywords');
|
|
29
|
+
t.json('excludeUrls');
|
|
30
|
+
t.integer('maxExcludedDepth');
|
|
31
|
+
t.integer('retry');
|
|
32
|
+
t.boolean('fromList');
|
|
33
|
+
t.boolean('disableQueries');
|
|
34
|
+
t.string('userAgent');
|
|
35
|
+
t.boolean('ignoreRobots');
|
|
36
|
+
})
|
|
37
|
+
.createTable('pages', (t) => {
|
|
38
|
+
t.increments('id');
|
|
39
|
+
t.string('url', 8190).notNullable().unique();
|
|
40
|
+
t.integer('redirectDestId').unsigned().references('pages.id').defaultTo(null);
|
|
41
|
+
t.boolean('scraped').notNullable();
|
|
42
|
+
t.boolean('isTarget').notNullable();
|
|
43
|
+
t.boolean('isExternal');
|
|
44
|
+
t.integer('status');
|
|
45
|
+
t.string('statusText');
|
|
46
|
+
t.string('contentType').nullable();
|
|
47
|
+
t.integer('contentLength').unsigned().nullable();
|
|
48
|
+
t.json('responseHeaders').nullable();
|
|
49
|
+
t.string('lang');
|
|
50
|
+
t.string('title');
|
|
51
|
+
t.string('description');
|
|
52
|
+
t.string('keywords');
|
|
53
|
+
t.boolean('noindex');
|
|
54
|
+
t.boolean('nofollow');
|
|
55
|
+
t.boolean('noarchive');
|
|
56
|
+
t.string('canonical');
|
|
57
|
+
t.string('alternate');
|
|
58
|
+
t.string('og_type');
|
|
59
|
+
t.string('og_title');
|
|
60
|
+
t.string('og_site_name');
|
|
61
|
+
t.string('og_description');
|
|
62
|
+
t.string('og_url');
|
|
63
|
+
t.string('og_image');
|
|
64
|
+
t.string('twitter_card');
|
|
65
|
+
t.string('html');
|
|
66
|
+
t.boolean('isSkipped');
|
|
67
|
+
t.string('skipReason');
|
|
68
|
+
t.integer('order').unsigned().nullable();
|
|
69
|
+
t.index('isExternal');
|
|
70
|
+
t.index('contentType');
|
|
71
|
+
t.index('scraped');
|
|
72
|
+
t.index('redirectDestId');
|
|
73
|
+
t.index('order');
|
|
74
|
+
})
|
|
75
|
+
.createTable('anchors', (t) => {
|
|
76
|
+
t.increments('id');
|
|
77
|
+
t.integer('pageId').notNullable().unsigned().references('pages.id');
|
|
78
|
+
t.integer('hrefId').notNullable().unsigned().references('pages.id');
|
|
79
|
+
t.string('hash');
|
|
80
|
+
t.string('textContent').nullable();
|
|
81
|
+
t.index('pageId');
|
|
82
|
+
t.index('hrefId');
|
|
83
|
+
})
|
|
84
|
+
.createTable('images', (t) => {
|
|
85
|
+
t.increments('id');
|
|
86
|
+
t.integer('pageId').notNullable().unsigned().references('pages.id');
|
|
87
|
+
t.string('src', 8190);
|
|
88
|
+
t.string('currentSrc', 8190);
|
|
89
|
+
t.string('alt');
|
|
90
|
+
t.float('width').unsigned().notNullable();
|
|
91
|
+
t.float('height').unsigned().notNullable();
|
|
92
|
+
t.integer('naturalWidth').unsigned().notNullable();
|
|
93
|
+
t.integer('naturalHeight').unsigned().notNullable();
|
|
94
|
+
t.boolean('isLazy');
|
|
95
|
+
t.integer('viewportWidth').unsigned().notNullable();
|
|
96
|
+
t.string('sourceCode');
|
|
97
|
+
t.index('pageId');
|
|
98
|
+
})
|
|
99
|
+
.createTable('resources', (t) => {
|
|
100
|
+
t.increments('id');
|
|
101
|
+
t.string('url', 8190).notNullable().unique();
|
|
102
|
+
t.boolean('isExternal');
|
|
103
|
+
t.integer('status');
|
|
104
|
+
t.string('statusText');
|
|
105
|
+
t.string('contentType').nullable();
|
|
106
|
+
t.integer('contentLength').unsigned().nullable();
|
|
107
|
+
t.string('compress').nullable();
|
|
108
|
+
t.string('cdn').nullable();
|
|
109
|
+
t.json('responseHeaders').nullable();
|
|
110
|
+
})
|
|
111
|
+
.createTable('resources-referrers', (t) => {
|
|
112
|
+
t.increments('id');
|
|
113
|
+
t.integer('resourceId').notNullable().unsigned().references('resources.id');
|
|
114
|
+
t.integer('pageId').notNullable().unsigned().references('pages.id');
|
|
115
|
+
t.unique(['resourceId', 'pageId']);
|
|
116
|
+
t.index('resourceId');
|
|
117
|
+
t.index('pageId');
|
|
118
|
+
});
|
|
119
|
+
}
|
|
@@ -0,0 +1,8 @@
|
|
|
1
|
+
import type { Knex } from 'knex';
|
|
2
|
+
/**
|
|
3
|
+
* Returns a Knex subquery builder that selects page IDs with pagination,
|
|
4
|
+
* ordered by the `order` column (nulls last), excluding redirected pages.
|
|
5
|
+
* @param limit - The maximum number of page IDs to return.
|
|
6
|
+
* @param offset - The number of page IDs to skip before returning results.
|
|
7
|
+
*/
|
|
8
|
+
export declare function limitedPageIds(limit: number, offset: number): (qb: Knex.QueryBuilder<Record<string, unknown>, unknown>) => Promise<void>;
|
|
@@ -0,0 +1,17 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Returns a Knex subquery builder that selects page IDs with pagination,
|
|
3
|
+
* ordered by the `order` column (nulls last), excluding redirected pages.
|
|
4
|
+
* @param limit - The maximum number of page IDs to return.
|
|
5
|
+
* @param offset - The number of page IDs to skip before returning results.
|
|
6
|
+
*/
|
|
7
|
+
export function limitedPageIds(limit, offset) {
|
|
8
|
+
return async (qb) => {
|
|
9
|
+
await qb
|
|
10
|
+
.select('id')
|
|
11
|
+
.from('pages')
|
|
12
|
+
.orderByRaw('`order` ASC NULLS LAST')
|
|
13
|
+
.whereNull('redirectDestId')
|
|
14
|
+
.limit(limit)
|
|
15
|
+
.offset(offset);
|
|
16
|
+
};
|
|
17
|
+
}
|
package/lib/archive/page.d.ts
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import type { ArchiveAccessor } from './archive-accessor.js';
|
|
2
|
-
import type { DB_Anchor, DB_Page, DB_Redirect, DB_Referrer } from './types.js';
|
|
2
|
+
import type { Anchor, Redirect, Referrer, DB_Anchor, DB_Page, DB_Redirect, DB_Referrer } from './types.js';
|
|
3
3
|
/**
|
|
4
4
|
* Represents a crawled page stored in the archive.
|
|
5
5
|
*
|
|
@@ -213,51 +213,4 @@ type PromiseType<T> = T extends PromiseLike<infer U> ? U : T;
|
|
|
213
213
|
* The static (serialized) representation of a Page, as returned by {@link Page.toJSON}.
|
|
214
214
|
*/
|
|
215
215
|
export type StaticPageData = PromiseType<ReturnType<Page['toJSON']>>;
|
|
216
|
-
/**
|
|
217
|
-
* Represents a page that links to another page (an incoming link).
|
|
218
|
-
*/
|
|
219
|
-
export interface Referrer {
|
|
220
|
-
/** The URL of the referring page. */
|
|
221
|
-
url: string;
|
|
222
|
-
/** The URL through which the referral passes (may differ due to redirects). */
|
|
223
|
-
through: string;
|
|
224
|
-
/** The page ID corresponding to the through URL. */
|
|
225
|
-
throughId: number;
|
|
226
|
-
/** The URL fragment (hash) of the referring link, or null if not present. */
|
|
227
|
-
hash: string | null;
|
|
228
|
-
/** The text content of the referring anchor element. */
|
|
229
|
-
textContent: string;
|
|
230
|
-
}
|
|
231
|
-
/**
|
|
232
|
-
* Represents an outgoing link (anchor element) found on a page.
|
|
233
|
-
*/
|
|
234
|
-
export interface Anchor {
|
|
235
|
-
/** The resolved destination URL of the anchor. */
|
|
236
|
-
url: string;
|
|
237
|
-
/** The original href attribute value of the anchor element. */
|
|
238
|
-
href: string;
|
|
239
|
-
/** Whether the anchor points to an external domain. */
|
|
240
|
-
isExternal: boolean;
|
|
241
|
-
/** The title attribute of the anchor element, or null if not present. */
|
|
242
|
-
title: string | null;
|
|
243
|
-
/** The HTTP status code of the linked page, or null if not yet fetched. */
|
|
244
|
-
status: number | null;
|
|
245
|
-
/** The HTTP status text of the linked page, or null if not yet fetched. */
|
|
246
|
-
statusText: string | null;
|
|
247
|
-
/** The content type of the linked page, or null if not yet fetched. */
|
|
248
|
-
contentType: string | null;
|
|
249
|
-
/** The URL fragment (hash) portion of the link, or null if not present. */
|
|
250
|
-
hash: string | null;
|
|
251
|
-
/** The text content of the anchor element, or null if empty. */
|
|
252
|
-
textContent: string | null;
|
|
253
|
-
}
|
|
254
|
-
/**
|
|
255
|
-
* Represents a page that redirects to this page.
|
|
256
|
-
*/
|
|
257
|
-
export interface Redirect {
|
|
258
|
-
/** The URL of the redirect source page. */
|
|
259
|
-
url: string;
|
|
260
|
-
/** The database ID of the redirect source page. */
|
|
261
|
-
pageId: number;
|
|
262
|
-
}
|
|
263
216
|
export {};
|
|
@@ -0,0 +1,7 @@
|
|
|
1
|
+
import type { Knex } from 'knex';
|
|
2
|
+
/**
|
|
3
|
+
* Returns a Knex subquery builder that joins pages with their redirect destinations.
|
|
4
|
+
* When `includeNull` is true, also includes pages without redirects (self-referencing).
|
|
5
|
+
* @param includeNull - Whether to include non-redirected pages in the result. Defaults to `true`.
|
|
6
|
+
*/
|
|
7
|
+
export declare function redirectTable(includeNull?: boolean): (qb: Knex.QueryBuilder<Record<string, unknown>, unknown>) => Promise<void>;
|
|
@@ -0,0 +1,23 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Returns a Knex subquery builder that joins pages with their redirect destinations.
|
|
3
|
+
* When `includeNull` is true, also includes pages without redirects (self-referencing).
|
|
4
|
+
* @param includeNull - Whether to include non-redirected pages in the result. Defaults to `true`.
|
|
5
|
+
*/
|
|
6
|
+
export function redirectTable(includeNull = true) {
|
|
7
|
+
return async (qb) => {
|
|
8
|
+
const list = qb
|
|
9
|
+
.select('A.id as fromId', 'A.url as from', 'B.url as to', 'B.id as toId')
|
|
10
|
+
.from('pages as A')
|
|
11
|
+
.join('pages as B', (j) => {
|
|
12
|
+
j.on('A.redirectDestId', '=', 'B.id').andOnNotNull('A.redirectDestId');
|
|
13
|
+
});
|
|
14
|
+
if (includeNull) {
|
|
15
|
+
await list.union(async (qb) => {
|
|
16
|
+
await qb
|
|
17
|
+
.select('A.id as fromId', 'A.url as from', 'A.url as to', 'A.id as toId')
|
|
18
|
+
.from('pages as A')
|
|
19
|
+
.whereNull('A.redirectDestId');
|
|
20
|
+
});
|
|
21
|
+
}
|
|
22
|
+
};
|
|
23
|
+
}
|
package/lib/archive/types.d.ts
CHANGED
|
@@ -212,6 +212,53 @@ export interface DB_Image {
|
|
|
212
212
|
/** The raw HTML source code of the image element. */
|
|
213
213
|
sourceCode: string | null;
|
|
214
214
|
}
|
|
215
|
+
/**
|
|
216
|
+
* Represents a page that links to another page (an incoming link).
|
|
217
|
+
*/
|
|
218
|
+
export interface Referrer {
|
|
219
|
+
/** The URL of the referring page. */
|
|
220
|
+
url: string;
|
|
221
|
+
/** The URL through which the referral passes (may differ due to redirects). */
|
|
222
|
+
through: string;
|
|
223
|
+
/** The page ID corresponding to the through URL. */
|
|
224
|
+
throughId: number;
|
|
225
|
+
/** The URL fragment (hash) of the referring link, or null if not present. */
|
|
226
|
+
hash: string | null;
|
|
227
|
+
/** The text content of the referring anchor element. */
|
|
228
|
+
textContent: string;
|
|
229
|
+
}
|
|
230
|
+
/**
|
|
231
|
+
* Represents an outgoing link (anchor element) found on a page.
|
|
232
|
+
*/
|
|
233
|
+
export interface Anchor {
|
|
234
|
+
/** The resolved destination URL of the anchor. */
|
|
235
|
+
url: string;
|
|
236
|
+
/** The original href attribute value of the anchor element. */
|
|
237
|
+
href: string;
|
|
238
|
+
/** Whether the anchor points to an external domain. */
|
|
239
|
+
isExternal: boolean;
|
|
240
|
+
/** The title attribute of the anchor element, or null if not present. */
|
|
241
|
+
title: string | null;
|
|
242
|
+
/** The HTTP status code of the linked page, or null if not yet fetched. */
|
|
243
|
+
status: number | null;
|
|
244
|
+
/** The HTTP status text of the linked page, or null if not yet fetched. */
|
|
245
|
+
statusText: string | null;
|
|
246
|
+
/** The content type of the linked page, or null if not yet fetched. */
|
|
247
|
+
contentType: string | null;
|
|
248
|
+
/** The URL fragment (hash) portion of the link, or null if not present. */
|
|
249
|
+
hash: string | null;
|
|
250
|
+
/** The text content of the anchor element, or null if empty. */
|
|
251
|
+
textContent: string | null;
|
|
252
|
+
}
|
|
253
|
+
/**
|
|
254
|
+
* Represents a page that redirects to this page.
|
|
255
|
+
*/
|
|
256
|
+
export interface Redirect {
|
|
257
|
+
/** The URL of the redirect source page. */
|
|
258
|
+
url: string;
|
|
259
|
+
/** The database ID of the redirect source page. */
|
|
260
|
+
pageId: number;
|
|
261
|
+
}
|
|
215
262
|
/**
|
|
216
263
|
* Raw database row representing a sub-resource (CSS, JS, image, etc.) in the `resources` table.
|
|
217
264
|
*/
|
|
@@ -237,3 +284,32 @@ export interface DB_Resource {
|
|
|
237
284
|
/** JSON-serialized HTTP response headers, or null if not available. */
|
|
238
285
|
responseHeaders: string | null;
|
|
239
286
|
}
|
|
287
|
+
/**
|
|
288
|
+
* Base options shared by all database connection configurations.
|
|
289
|
+
*/
|
|
290
|
+
type AbsDatabaseOption = {
|
|
291
|
+
/** The working directory for the database (used for resolving relative paths). */
|
|
292
|
+
workingDir: string;
|
|
293
|
+
};
|
|
294
|
+
/**
|
|
295
|
+
* Union type for all supported database connection options.
|
|
296
|
+
*/
|
|
297
|
+
export type DatabaseOption = DatabaseSqlite3Option | DatabaseMySqlOption;
|
|
298
|
+
/**
|
|
299
|
+
* Connection options for a SQLite3 database.
|
|
300
|
+
*/
|
|
301
|
+
type DatabaseSqlite3Option = AbsDatabaseOption & {
|
|
302
|
+
/** The database type identifier. */
|
|
303
|
+
type: 'sqlite3';
|
|
304
|
+
/** The absolute file path to the SQLite database file. */
|
|
305
|
+
filename: string;
|
|
306
|
+
};
|
|
307
|
+
/**
|
|
308
|
+
* Connection options for a MySQL database.
|
|
309
|
+
* Note: MySQL support is not yet implemented.
|
|
310
|
+
*/
|
|
311
|
+
type DatabaseMySqlOption = AbsDatabaseOption & {
|
|
312
|
+
/** The database type identifier. */
|
|
313
|
+
type: 'mysql';
|
|
314
|
+
};
|
|
315
|
+
export {};
|
package/lib/crawler.d.ts
CHANGED
|
@@ -12,7 +12,7 @@ export type { ErrorEvent } from './utils/error/error-emitter.js';
|
|
|
12
12
|
export * from './utils/object/clean-object.js';
|
|
13
13
|
export { globalLog as log } from './utils/debug.js';
|
|
14
14
|
export { ArchiveAccessor } from './archive/archive-accessor.js';
|
|
15
|
-
export type {
|
|
15
|
+
export type { StaticPageData } from './archive/page.js';
|
|
16
16
|
export { default as Page } from './archive/page.js';
|
|
17
17
|
export { default as ArchiveResource } from './archive/resource.js';
|
|
18
18
|
export * from './archive/types.js';
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@nitpicker/crawler",
|
|
3
|
-
"version": "0.6.
|
|
3
|
+
"version": "0.6.5-alpha.0",
|
|
4
4
|
"description": "Web crawler engine with headless browser rendering and archive storage",
|
|
5
5
|
"author": "D-ZERO",
|
|
6
6
|
"license": "Apache-2.0",
|
|
@@ -48,5 +48,5 @@
|
|
|
48
48
|
"@types/tar": "7.0.87",
|
|
49
49
|
"@types/unzipper": "0.10.11"
|
|
50
50
|
},
|
|
51
|
-
"gitHead": "
|
|
51
|
+
"gitHead": "e084aba5a0887a80059ff8aa0608f61a58cc9288"
|
|
52
52
|
}
|
|
@@ -1,17 +0,0 @@
|
|
|
1
|
-
export { outputJSON } from './output-json.js';
|
|
2
|
-
export { readJSON } from './read-json.js';
|
|
3
|
-
export { outputText } from './output-text.js';
|
|
4
|
-
export { appendText } from './append-text.js';
|
|
5
|
-
export { readText } from './read-text.js';
|
|
6
|
-
export { copyDir } from './copy-dir.js';
|
|
7
|
-
export { copyDirSync } from './copy-dir-sync.js';
|
|
8
|
-
export { isDir } from './is-dir.js';
|
|
9
|
-
export { remove } from './remove.js';
|
|
10
|
-
export { rename } from './rename.js';
|
|
11
|
-
export { getFileList } from './get-file-list.js';
|
|
12
|
-
export { readline } from './readline.js';
|
|
13
|
-
export { mkdir } from './mkdir.js';
|
|
14
|
-
export { exists } from './exists.js';
|
|
15
|
-
export { tar } from './tar.js';
|
|
16
|
-
export { untar } from './untar.js';
|
|
17
|
-
export { zip, unzip, extractZip } from '@d-zero/fs/zip';
|