@arcblock/crawler 1.1.6 → 1.3.0

package/lib/esm/index.d.ts

@@ -3,4 +3,5 @@ export * from './crawler';
 export * from './site';
 export * from './services/snapshot';
 export * as utils from './utils';
+export { migrate } from './store/migrate';
 export declare function initCrawler(params: Pick<Config, 'puppeteerPath' | 'siteCron' | 'cookies' | 'localStorage' | 'concurrency'>): Promise<void>;

package/lib/esm/index.js

@@ -10,23 +10,20 @@ var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, ge
 /* eslint-disable @typescript-eslint/indent */
 import merge from 'lodash/merge';
 import { config, logger } from './config';
-import { createCrawlQueue } from './crawler';
 import { initCron } from './cron';
 import { ensureBrowser } from './puppeteer';
-import { initDatabase } from './store';
 export * from './crawler';
 export * from './site';
 export * from './services/snapshot';
 export * as utils from './utils';
+export { migrate } from './store/migrate';
 export function initCrawler(params) {
     return __awaiter(this, void 0, void 0, function* () {
         var _a;
         merge(config, params);
         logger.info('Init crawler', { params, config });
         try {
-            yield initDatabase();
             yield ensureBrowser();
-            yield createCrawlQueue();
             if ((_a = config.siteCron) === null || _a === void 0 ? void 0 : _a.enabled) {
                 yield initCron();
             }

package/lib/esm/services/carbon.d.ts

@@ -0,0 +1,3 @@
+import { Page } from '@blocklet/puppeteer';
+import { JobState } from '../store';
+export declare function createCarbonImage(page: Page, params?: JobState): Promise<Buffer<ArrayBuffer>>;

package/lib/esm/services/carbon.js

@@ -0,0 +1,84 @@
+var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
+    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
+    return new (P || (P = Promise))(function (resolve, reject) {
+        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
+        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
+        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
+        step((generator = generator.apply(thisArg, _arguments || [])).next());
+    });
+};
+import { logger } from '../config';
+// TODO expose local version of dom-to-image
+const DOM_TO_IMAGE_URL = 'https://unpkg.com/dom-to-image@2.6.0/dist/dom-to-image.min.js';
+export function createCarbonImage(page, params) {
+    return __awaiter(this, void 0, void 0, function* () {
+        try {
+            yield page.addScriptTag({ url: DOM_TO_IMAGE_URL });
+            yield page.waitForSelector('.export-container', { visible: true, timeout: (params === null || params === void 0 ? void 0 : params.timeout) || 120 });
+            const targetElement = yield page.$('.export-container');
+            const format = (params === null || params === void 0 ? void 0 : params.format) || 'png';
+            const dataUrl = yield page.evaluate((target = document, imageFormat = 'png') => {
+                const query = new URLSearchParams(document.location.search);
+                const EXPORT_SIZES_HASH = {
+                    '1x': '1',
+                    '2x': '2',
+                    '4x': '4',
+                };
+                const exportSize = EXPORT_SIZES_HASH[query.get('es')] || '2';
+                if (!target) {
+                    throw new Error('Target element not found');
+                }
+                target.querySelectorAll('span[role="presentation"]').forEach((node) => {
+                    var _a;
+                    const el = node;
+                    if (el && el.innerText && el.innerText.match(/%[A-Fa-f0-9]{2}/)) {
+                        (_a = el.innerText.match(/%[A-Fa-f0-9]{2}/g)) === null || _a === void 0 ? void 0 : _a.forEach((t) => {
+                            el.innerHTML = el.innerHTML.replace(t, encodeURIComponent(t));
+                        });
+                    }
+                });
+                const width = target.offsetWidth * exportSize;
+                const height = query.get('si') === 'true'
+                    ? target.offsetWidth * exportSize
+                    : target.offsetHeight * exportSize;
+                const config = {
+                    style: {
+                        transform: `scale(${exportSize})`,
+                        'transform-origin': 'center',
+                        background: query.get('si') ? query.get('bg') : 'none',
+                    },
+                    filter: (n) => {
+                        if (n.className) {
+                            return String(n.className).indexOf('eliminateOnRender') < 0;
+                        }
+                        return true;
+                    },
+                    width,
+                    height,
+                };
+                switch (imageFormat) {
+                    case 'jpeg':
+                        // @ts-ignore: domtoimage is injected by addScriptTag
+                        return domtoimage.toJpeg(target, config);
+                    case 'webp':
+                        // dom-to-image doesn't support webp directly, fall back to png
+                        // @ts-ignore: domtoimage is injected by addScriptTag
+                        return domtoimage.toPng(target, config);
+                    case 'png':
+                    default:
+                        // @ts-ignore: domtoimage is injected by addScriptTag
+                        return domtoimage.toPng(target, config);
+                }
+            }, targetElement, format);
+            const base64Data = dataUrl.split(',')[1];
+            if (!base64Data) {
+                throw new Error('Failed to extract base64 data from image');
+            }
+            return Buffer.from(base64Data, 'base64');
+        }
+        catch (e) {
+            logger.error('failed to crawl from carbon', { error: e });
+            throw e;
+        }
+    });
+}

package/lib/esm/services/snapshot.d.ts

@@ -1,5 +1,5 @@
-import { JobState } from '../store/job';
-import { SnapshotModel } from '../store/snapshot';
+import { Transaction, WhereOptions } from '@sequelize/core';
+import { JobState, SnapshotModel } from '../store';
 export declare function convertJobToSnapshot({ job, snapshot }: {
     job: JobState;
     snapshot?: Partial<SnapshotModel>;
@@ -10,3 +10,6 @@ export declare function formatSnapshot(snapshot: SnapshotModel, columns?: Array<
  */
 export declare function getSnapshot(jobId: string): Promise<SnapshotModel | null>;
 export declare function getLatestSnapshot(url: string): Promise<SnapshotModel | null>;
+export declare function deleteSnapshots(where: WhereOptions<SnapshotModel>, { txn }?: {
+    txn?: Transaction;
+}): Promise<string[]>;

package/lib/esm/services/snapshot.js

@@ -12,12 +12,11 @@ import pick from 'lodash/pick';
 import fs from 'node:fs/promises';
 import path from 'node:path';
 import { joinURL } from 'ufo';
-import { config } from '../config';
-import { Job } from '../store/job';
-import { Snapshot } from '../store/snapshot';
+import { config, logger } from '../config';
+import { Job, Snapshot } from '../store';
 import { formatUrl } from '../utils';
 export function convertJobToSnapshot({ job, snapshot }) {
-    return Object.assign({ jobId: job.jobId || job.id, url: job.url, lastModified: job.lastModified || new Date().toISOString(), options: {
+    return Object.assign({ jobId: job.jobId || job.id, url: job.url, lastModified: job.lastModified || new Date().toISOString(), replace: job.replace, options: {
             width: job.width,
             height: job.height,
             includeScreenshot: job.includeScreenshot,
@@ -76,7 +75,37 @@ export function getLatestSnapshot(url) {
                 url: formatUrl(url),
                 status: 'success',
             },
+            order: [
+                ['lastModified', 'DESC'],
+                ['updatedAt', 'DESC'],
+            ],
         });
         return snapshot ? formatSnapshot(snapshot) : null;
     });
 }
+export function deleteSnapshots(where_1) {
+    return __awaiter(this, arguments, void 0, function* (where, { txn } = {}) {
+        const snapshots = yield Snapshot.findAll({
+            where,
+            order: [
+                ['lastModified', 'DESC'],
+                ['updatedAt', 'DESC'],
+            ],
+        });
+        const jobIds = yield Promise.all(snapshots.map((snapshot) => __awaiter(this, void 0, void 0, function* () {
+            try {
+                yield Promise.all([
+                    snapshot.html && fs.unlink(path.join(config.dataDir, snapshot.html)),
+                    snapshot.screenshot && fs.unlink(path.join(config.dataDir, snapshot.screenshot)),
+                ]);
+                yield snapshot.destroy({ transaction: txn });
+                return snapshot.jobId;
+            }
+            catch (error) {
+                logger.error('Failed to delete snapshot', { error, snapshot });
+                throw error;
+            }
+        })));
+        return jobIds.filter(Boolean);
+    });
+}

package/lib/esm/site.d.ts

@@ -1,2 +1,2 @@
 import { Site } from './config';
-export declare const crawlSite: ({ url, pathname, interval }: Site) => Promise<(string | null)[]>;
+export declare const crawlSite: ({ url, pathname, interval }: Site) => Promise<(`${string}-${string}-${string}-${string}-${string}` | null)[]>;

package/lib/esm/site.js

@@ -8,12 +8,14 @@ var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, ge
     });
 };
 import uniq from 'lodash/uniq';
+import { randomUUID } from 'node:crypto';
 import pMap from 'p-map';
 import { config, logger } from './config';
-import { crawlUrl } from './crawler';
-import { Snapshot } from './store/snapshot';
+import { createCrawlQueue } from './crawler';
+import { Snapshot } from './store';
 import { formatUrl, getSitemapList } from './utils';
 const crawlBlockletRunningMap = new Map();
+const crawlQueue = createCrawlQueue('cronJobs');
 function parseSitemapUrl(sitemapItem) {
     var _a;
     const links = ((_a = sitemapItem.links) === null || _a === void 0 ? void 0 : _a.map((item) => item.url)) || [];
@@ -61,12 +63,16 @@ export const crawlSite = (_a) => __awaiter(void 0, [_a], void 0, function* ({ ur
                 url,
             });
             crawlCount++;
-            return crawlUrl({
+            const jobId = randomUUID();
+            crawlQueue.push({
+                id: jobId,
                 url,
                 lastModified: sitemapItem.lastmod,
                 includeScreenshot: false,
                 includeHtml: true,
+                replace: true,
             });
+            return jobId;
         }), { concurrency: ((_b = config.siteCron) === null || _b === void 0 ? void 0 : _b.concurrency) || 30 });
         logger.info('Enqueued jobs from sitemap finished', {
             url,

package/lib/esm/store/index.d.ts

@@ -1,3 +1,6 @@
 import { Sequelize } from '@sequelize/core';
 import { SqliteDialect } from '@sequelize/sqlite3';
-export declare function initDatabase(): Promise<Sequelize<SqliteDialect>>;
+declare const sequelize: Sequelize<SqliteDialect>;
+export { sequelize };
+export * from './job';
+export * from './snapshot';

package/lib/esm/store/index.js

@@ -1,51 +1,29 @@
-var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
-    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
-    return new (P || (P = Promise))(function (resolve, reject) {
-        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
-        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
-        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
-        step((generator = generator.apply(thisArg, _arguments || [])).next());
-    });
-};
 import { Sequelize } from '@sequelize/core';
 import { SqliteDialect } from '@sequelize/sqlite3';
 import path from 'path';
 import { config, logger } from '../config';
 import { Job } from './job';
 import { Snapshot } from './snapshot';
-export function initDatabase() {
-    return __awaiter(this, void 0, void 0, function* () {
-        const sequelize = new Sequelize({
-            dialect: SqliteDialect,
-            storage: path.join(config.dataDir, 'snap-kit.db'),
-            logging: (msg) => process.env.SQLITE_LOG && logger.debug(msg),
-            pool: {
-                min: 0,
-                max: 10,
-                idle: 10000,
-            },
-            retry: {
-                match: [/SQLITE_BUSY/],
-                name: 'query',
-                max: 10,
-            },
-        });
-        Job.initModel(sequelize);
-        Snapshot.initModel(sequelize);
-        try {
-            yield Promise.all([
-                sequelize.query('pragma journal_mode = WAL;'),
-                sequelize.query('pragma synchronous = normal;'),
-                sequelize.query('pragma journal_size_limit = 67108864;'),
-            ]);
-            yield sequelize.authenticate();
-            yield sequelize.sync({ alter: process.env.ALTER_SQLITE === 'true' });
-            logger.info('Successfully connected to database');
-        }
-        catch (error) {
-            logger.error('Failed to connect to database:', error);
-            throw error;
-        }
-        return sequelize;
-    });
-}
+const sequelize = new Sequelize({
+    dialect: SqliteDialect,
+    storage: path.join(config.dataDir, 'snap-kit.db'),
+    logging: (msg) => process.env.SQLITE_LOG && logger.debug(msg),
+    pool: {
+        min: 0,
+        max: 10,
+        idle: 10000,
+    },
+    retry: {
+        match: [/SQLITE_BUSY/],
+        name: 'query',
+        max: 10,
+    },
+});
+sequelize.query('pragma journal_mode = WAL;');
+sequelize.query('pragma synchronous = normal;');
+sequelize.query('pragma journal_size_limit = 67108864;');
+Job.initModel(sequelize);
+Snapshot.initModel(sequelize);
+export { sequelize };
+export * from './job';
+export * from './snapshot';

package/lib/esm/store/job.d.ts

@@ -9,9 +9,14 @@ export interface JobState {
     width?: number;
     height?: number;
     quality?: number;
+    format?: 'png' | 'jpeg' | 'webp';
     timeout?: number;
     fullPage?: boolean;
     lastModified?: string;
+    waitTime?: number;
+    replace?: boolean;
+    sync?: boolean;
+    ignoreRobots?: boolean;
     headers?: Record<string, string>;
     cookies?: CookieParam[];
     localStorage?: {

package/lib/esm/store/migrate.d.ts

@@ -0,0 +1,4 @@
+import { Umzug } from 'umzug';
+declare const umzug: Umzug<import("@sequelize/sqlite3").SqliteQueryInterface<import("@sequelize/sqlite3").SqliteDialect>>;
+export declare function migrate(): Promise<import("umzug").MigrationMeta[]>;
+export { umzug };

package/lib/esm/store/migrate.js

@@ -0,0 +1,26 @@
+/* eslint-disable global-require */
+import { SequelizeStorage, Umzug } from 'umzug';
+import { sequelize } from './index';
+import * as migration20250615 from './migrations/20250615-genesis';
+import * as migration20250616Replace from './migrations/20250616-replace';
+const umzug = new Umzug({
+    migrations: [
+        {
+            name: '20250615-genesis',
+            up: ({ context }) => migration20250615.up({ context }),
+            down: ({ context }) => migration20250615.down({ context }),
+        },
+        {
+            name: '20250616-replace',
+            up: ({ context }) => migration20250616Replace.up({ context }),
+            down: ({ context }) => migration20250616Replace.down({ context }),
+        },
+    ],
+    context: sequelize.getQueryInterface(),
+    storage: new SequelizeStorage({ sequelize }),
+    logger: console,
+});
+export function migrate() {
+    return umzug.up();
+}
+export { umzug };

package/lib/esm/store/migrations/20250615-genesis.d.ts

@@ -0,0 +1,6 @@
+export declare function up({ context }: {
+    context: any;
+}): Promise<void>;
+export declare function down({ context }: {
+    context: any;
+}): Promise<void>;

package/lib/esm/store/migrations/20250615-genesis.js

@@ -0,0 +1,110 @@
+var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
+    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
+    return new (P || (P = Promise))(function (resolve, reject) {
+        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
+        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
+        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
+        step((generator = generator.apply(thisArg, _arguments || [])).next());
+    });
+};
+/* eslint-disable no-console */
+import { DataTypes } from '@sequelize/core';
+export function up(_a) {
+    return __awaiter(this, arguments, void 0, function* ({ context }) {
+        console.log('[20250615-genesis:up] Migrating...');
+        yield context.createTable('snap', {
+            jobId: {
+                type: DataTypes.STRING,
+                primaryKey: true,
+                allowNull: false,
+            },
+            url: {
+                type: DataTypes.STRING,
+                allowNull: false,
+                index: true,
+            },
+            status: {
+                type: DataTypes.ENUM('success', 'failed', 'pending'),
+                allowNull: false,
+            },
+            html: {
+                type: DataTypes.TEXT,
+                allowNull: true,
+            },
+            screenshot: {
+                type: DataTypes.STRING,
+                allowNull: true,
+            },
+            error: {
+                type: DataTypes.STRING,
+                allowNull: true,
+            },
+            lastModified: {
+                type: DataTypes.STRING,
+                allowNull: true,
+            },
+            meta: {
+                type: DataTypes.JSON,
+                allowNull: true,
+            },
+            options: {
+                type: DataTypes.JSON,
+                allowNull: true,
+            },
+            createdAt: {
+                type: DataTypes.DATE,
+                defaultValue: DataTypes.NOW,
+            },
+            updatedAt: {
+                type: DataTypes.DATE,
+                defaultValue: DataTypes.NOW,
+            },
+        });
+        yield context.createTable('jobs', {
+            id: {
+                type: DataTypes.STRING(40),
+                primaryKey: true,
+            },
+            queue: {
+                type: DataTypes.STRING(32),
+                allowNull: false,
+            },
+            job: {
+                type: DataTypes.JSON,
+                allowNull: false,
+            },
+            retryCount: {
+                type: DataTypes.INTEGER,
+            },
+            delay: {
+                type: DataTypes.INTEGER,
+            },
+            willRunAt: {
+                type: DataTypes.INTEGER,
+            },
+            cancelled: {
+                type: DataTypes.BOOLEAN,
+                defaultValue: false,
+            },
+            createdAt: {
+                type: DataTypes.DATE,
+                defaultValue: DataTypes.NOW,
+                index: true,
+            },
+            updatedAt: {
+                type: DataTypes.DATE,
+                defaultValue: DataTypes.NOW,
+                index: true,
+            },
+        });
+        console.log('[20250615-genesis:up] Migrated successfully!');
+    });
+}
+export function down(_a) {
+    return __awaiter(this, arguments, void 0, function* ({ context }) {
+        console.log('[20250615-genesis:down] Migrating...');
+        yield context.dropTable('snap');
+        yield context.dropTable('jobs');
+        console.log('[20250615-genesis:down] Migrated successfully!');
+    });
+}

package/lib/esm/store/migrations/20250616-replace.d.ts

@@ -0,0 +1,6 @@
+export declare function up({ context }: {
+    context: any;
+}): Promise<void>;
+export declare function down({ context }: {
+    context: any;
+}): Promise<void>;

package/lib/esm/store/migrations/20250616-replace.js

@@ -0,0 +1,36 @@
+var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
+    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
+    return new (P || (P = Promise))(function (resolve, reject) {
+        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
+        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
+        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
+        step((generator = generator.apply(thisArg, _arguments || [])).next());
+    });
+};
+/* eslint-disable no-console */
+import { DataTypes } from '@sequelize/core';
+export function up(_a) {
+    return __awaiter(this, arguments, void 0, function* ({ context }) {
+        console.log('[20250616-replace:up] Migrating...');
+        yield context.addColumn('snap', 'replace', {
+            type: DataTypes.BOOLEAN,
+            allowNull: false,
+            defaultValue: false,
+            index: true,
+        });
+        yield context.addIndex('snap', ['createdAt']);
+        yield context.addIndex('snap', ['updatedAt']);
+        yield context.addIndex('snap', ['status']);
+        console.log('[20250616-replace:up] Migrated successfully!');
+    });
+}
+export function down(_a) {
+    return __awaiter(this, arguments, void 0, function* ({ context }) {
+        console.log('[20250616-replace:down] Migrating...');
+        yield context.removeColumn('snap', 'replace');
+        yield context.removeIndex('snap', ['createdAt']);
+        yield context.removeIndex('snap', ['updatedAt']);
+        yield context.removeIndex('snap', ['status']);
+        console.log('[20250616-replace:down] Migrated successfully!');
+    });
+}

package/lib/esm/store/snapshot.d.ts

@@ -8,6 +8,7 @@ export interface SnapshotModel {
     screenshot?: string | null;
     error?: string;
     lastModified?: string;
+    replace?: boolean;
     meta?: {
         title?: string;
         description?: string;
@@ -35,6 +36,7 @@ export declare class Snapshot extends Model<SnapshotModel> implements SnapshotMo
     screenshot?: SnapshotModel['screenshot'];
     error?: SnapshotModel['error'];
     lastModified?: SnapshotModel['lastModified'];
+    replace?: SnapshotModel['replace'];
     meta?: SnapshotModel['meta'];
     options: SnapshotModel['options'];
     static initModel(sequelize: Sequelize): typeof Snapshot;

package/lib/esm/store/snapshot.js

@@ -24,6 +24,7 @@ export class Snapshot extends Model {
             status: {
                 type: DataTypes.ENUM('success', 'failed', 'pending'),
                 allowNull: false,
+                index: true,
             },
             html: {
                 type: DataTypes.TEXT,
@@ -41,6 +42,12 @@ export class Snapshot extends Model {
                 type: DataTypes.STRING,
                 allowNull: true,
             },
+            replace: {
+                type: DataTypes.BOOLEAN,
+                allowNull: false,
+                defaultValue: false,
+                index: true,
+            },
             meta: {
                 type: DataTypes.JSON,
                 allowNull: true,

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@arcblock/crawler",
-  "version": "1.1.6",
+  "version": "1.3.0",
   "main": "lib/cjs/index.js",
   "module": "lib/esm/index.js",
   "types": "lib/cjs/index.d.ts",
@@ -61,7 +61,8 @@
     "robots-parser": "^3.0.1",
     "sitemap": "^7.1.2",
     "sqlite3": "^5.1.7",
-    "ufo": "^1.5.4"
+    "ufo": "^1.5.4",
+    "umzug": "^3.8.2"
   },
   "devDependencies": {
     "@types/dotenv-flow": "^3.3.3",