@nitpicker/crawler 0.4.2 → 0.4.3

package/src/archive/debug.ts

@@ -1,10 +0,0 @@
-import { log as globalLog } from '../utils/debug.js';
-
-/** Debug logger for the archive package. Namespace: `Nitpicker:Utils:Archive`. */
-export const log = globalLog.extend('Archive');
-/** Debug logger for archive save operations. Namespace: `Nitpicker:Utils:Archive:Save`. */
-export const saveLog = log.extend('Save');
-/** Debug logger for database operations. Namespace: `Nitpicker:Utils:Archive:DB`. */
-export const dbLog = log.extend('DB');
-/** Debug logger for archive errors. Namespace: `Nitpicker:Utils:Archive:Error`. */
-export const errorLog = log.extend('Error');

package/src/archive/filesystem/append-text.spec.ts

@@ -1,26 +0,0 @@
-import { writeFileSync, readFileSync, mkdirSync, rmSync } from 'node:fs';
-import { tmpdir } from 'node:os';
-import path from 'node:path';
-
-import { afterEach, beforeEach, describe, expect, it } from 'vitest';
-
-import { appendText } from './append-text.js';
-
-describe('appendText', () => {
-	const testDir = path.join(tmpdir(), 'nitpicker-test-append-text');
-
-	beforeEach(() => {
-		mkdirSync(testDir, { recursive: true });
-	});
-
-	afterEach(() => {
-		rmSync(testDir, { recursive: true, force: true });
-	});
-
-	it('appends text with a preceding newline', async () => {
-		const filePath = path.join(testDir, 'file.txt');
-		writeFileSync(filePath, 'line1');
-		await appendText(filePath, 'line2');
-		expect(readFileSync(filePath, 'utf8')).toBe('line1\nline2');
-	});
-});

package/src/archive/filesystem/append-text.ts

@@ -1,16 +0,0 @@
-import { promises as fs } from 'node:fs';
-
-import { mkdir } from './mkdir.js';
-
-/**
- * Appends text data to a file at the specified path.
- *
- * Creates parent directories if they do not exist.
- * A newline character is prepended to the data before appending.
- * @param filePath - The absolute or relative path to the file to append to.
- * @param data - The text content to append to the file.
- */
-export async function appendText(filePath: string, data: string) {
-	mkdir(filePath);
-	await fs.appendFile(filePath, `\n${data}`, { encoding: 'utf8' });
-}

package/src/archive/filesystem/copy-dir-sync.spec.ts

@@ -1,27 +0,0 @@
-import { writeFileSync, readFileSync, mkdirSync, rmSync } from 'node:fs';
-import { tmpdir } from 'node:os';
-import path from 'node:path';
-
-import { afterEach, beforeEach, describe, expect, it } from 'vitest';
-
-import { copyDirSync } from './copy-dir-sync.js';
-
-describe('copyDirSync', () => {
-	const testDir = path.join(tmpdir(), 'nitpicker-test-copy-dir-sync');
-	const srcDir = path.join(testDir, 'src');
-	const destDir = path.join(testDir, 'dest');
-
-	beforeEach(() => {
-		mkdirSync(srcDir, { recursive: true });
-	});
-
-	afterEach(() => {
-		rmSync(testDir, { recursive: true, force: true });
-	});
-
-	it('synchronously copies directory contents', () => {
-		writeFileSync(path.join(srcDir, 'a.txt'), 'sync');
-		copyDirSync(srcDir, destDir);
-		expect(readFileSync(path.join(destDir, 'a.txt'), 'utf8')).toBe('sync');
-	});
-});

package/src/archive/filesystem/copy-dir-sync.ts

@@ -1,10 +0,0 @@
-import fsx from 'fs-extra';
-
-/**
- * Synchronously copies a directory and its contents from one location to another.
- * @param from - The source directory path to copy from.
- * @param to - The destination directory path to copy to.
- */
-export function copyDirSync(from: string, to: string) {
-	fsx.copySync(from, to);
-}

package/src/archive/filesystem/copy-dir.spec.ts

@@ -1,33 +0,0 @@
-import { writeFileSync, readFileSync, mkdirSync, rmSync } from 'node:fs';
-import { tmpdir } from 'node:os';
-import path from 'node:path';
-
-import { afterEach, beforeEach, describe, expect, it } from 'vitest';
-
-import { copyDir } from './copy-dir.js';
-
-describe('copyDir', () => {
-	const testDir = path.join(tmpdir(), 'nitpicker-test-copy-dir');
-	const srcDir = path.join(testDir, 'src');
-	const destDir = path.join(testDir, 'dest');
-
-	beforeEach(() => {
-		mkdirSync(srcDir, { recursive: true });
-	});
-
-	afterEach(() => {
-		rmSync(testDir, { recursive: true, force: true });
-	});
-
-	it('copies directory contents', async () => {
-		writeFileSync(path.join(srcDir, 'a.txt'), 'hello');
-		const result = await copyDir(srcDir, destDir);
-		expect(result).toBe(true);
-		expect(readFileSync(path.join(destDir, 'a.txt'), 'utf8')).toBe('hello');
-	});
-
-	it('returns false on error', async () => {
-		const result = await copyDir('/nonexistent/path', destDir);
-		expect(result).toBe(false);
-	});
-});

package/src/archive/filesystem/copy-dir.ts

@@ -1,14 +0,0 @@
-import fsx from 'fs-extra';
-
-/**
- * Recursively copies a directory and its contents from one location to another.
- * @param from - The source directory path to copy from.
- * @param to - The destination directory path to copy to.
- * @returns `true` if the copy succeeded, `false` if an error occurred.
- */
-export async function copyDir(from: string, to: string) {
-	return fsx
-		.copy(from, to)
-		.then(() => true)
-		.catch(() => false);
-}

package/src/archive/filesystem/exists.spec.ts

@@ -1,33 +0,0 @@
-import { writeFileSync, mkdirSync, rmSync } from 'node:fs';
-import { tmpdir } from 'node:os';
-import path from 'node:path';
-
-import { afterEach, beforeEach, describe, expect, it } from 'vitest';
-
-import { exists } from './exists.js';
-
-describe('exists', () => {
-	const testDir = path.join(tmpdir(), 'nitpicker-test-exists');
-
-	beforeEach(() => {
-		mkdirSync(testDir, { recursive: true });
-	});
-
-	afterEach(() => {
-		rmSync(testDir, { recursive: true, force: true });
-	});
-
-	it('returns true for an existing file', () => {
-		const filePath = path.join(testDir, 'file.txt');
-		writeFileSync(filePath, 'test');
-		expect(exists(filePath)).toBe(true);
-	});
-
-	it('returns false for a non-existing file', () => {
-		expect(exists(path.join(testDir, 'nonexistent.txt'))).toBe(false);
-	});
-
-	it('returns true for an existing directory', () => {
-		expect(exists(testDir)).toBe(true);
-	});
-});

package/src/archive/filesystem/exists.ts

@@ -1,10 +0,0 @@
-import { existsSync } from 'node:fs';
-
-/**
- * Checks whether a file or directory exists at the given path.
- * @param filePath - The path to check for existence.
- * @returns `true` if the path exists, `false` otherwise.
- */
-export function exists(filePath: string) {
-	return existsSync(filePath);
-}

package/src/archive/filesystem/get-file-list.spec.ts

@@ -1,37 +0,0 @@
-import { writeFileSync, mkdirSync, rmSync } from 'node:fs';
-import { tmpdir } from 'node:os';
-import path from 'node:path';
-
-import { afterEach, beforeEach, describe, expect, it } from 'vitest';
-
-import { getFileList } from './get-file-list.js';
-
-describe('getFileList', () => {
-	const testDir = path.join(tmpdir(), 'nitpicker-test-get-file-list');
-
-	beforeEach(() => {
-		mkdirSync(testDir, { recursive: true });
-		writeFileSync(path.join(testDir, 'a.txt'), '');
-		writeFileSync(path.join(testDir, 'b.json'), '');
-		writeFileSync(path.join(testDir, 'c.txt'), '');
-	});
-
-	afterEach(() => {
-		rmSync(testDir, { recursive: true, force: true });
-	});
-
-	it('lists all files without filter', async () => {
-		const list = await getFileList(testDir);
-		expect(list.toSorted()).toEqual(['a.txt', 'b.json', 'c.txt']);
-	});
-
-	it('filters by regex', async () => {
-		const list = await getFileList(testDir, /\.txt$/);
-		expect(list.toSorted()).toEqual(['a.txt', 'c.txt']);
-	});
-
-	it('filters by string', async () => {
-		const list = await getFileList(testDir, '.json');
-		expect(list).toEqual(['b.json']);
-	});
-});

package/src/archive/filesystem/get-file-list.ts

@@ -1,13 +0,0 @@
-import fsx from 'fs-extra';
-
-/**
- * Lists the file names in a directory, optionally filtered by a pattern.
- * @param dirPath - The directory path to list files from.
- * @param filter - An optional RegExp or string pattern to filter file names.
- *   Only file names matching this pattern are included in the result.
- * @returns An array of file names in the directory that match the filter (or all if no filter is provided).
- */
-export async function getFileList(dirPath: string, filter?: RegExp | string) {
-	const list = await fsx.readdir(dirPath);
-	return filter ? list.filter((fileName) => fileName.match(filter)) : list;
-}

package/src/archive/filesystem/index.ts

@@ -1,17 +0,0 @@
-export { outputJSON } from './output-json.js';
-export { readJSON } from './read-json.js';
-export { outputText } from './output-text.js';
-export { appendText } from './append-text.js';
-export { readText } from './read-text.js';
-export { copyDir } from './copy-dir.js';
-export { copyDirSync } from './copy-dir-sync.js';
-export { isDir } from './is-dir.js';
-export { remove } from './remove.js';
-export { rename } from './rename.js';
-export { getFileList } from './get-file-list.js';
-export { readline } from './readline.js';
-export { mkdir } from './mkdir.js';
-export { exists } from './exists.js';
-export { tar } from './tar.js';
-export { untar } from './untar.js';
-export { zip, unzip, extractZip } from '@d-zero/fs/zip';

package/src/archive/filesystem/is-dir.spec.ts

@@ -1,29 +0,0 @@
-import { writeFileSync, mkdirSync, rmSync } from 'node:fs';
-import { tmpdir } from 'node:os';
-import path from 'node:path';
-
-import { afterEach, beforeEach, describe, expect, it } from 'vitest';
-
-import { isDir } from './is-dir.js';
-
-describe('isDir', () => {
-	const testDir = path.join(tmpdir(), 'nitpicker-test-is-dir');
-
-	beforeEach(() => {
-		mkdirSync(testDir, { recursive: true });
-	});
-
-	afterEach(() => {
-		rmSync(testDir, { recursive: true, force: true });
-	});
-
-	it('returns true for a directory', async () => {
-		expect(await isDir(testDir)).toBe(true);
-	});
-
-	it('returns false for a file', async () => {
-		const filePath = path.join(testDir, 'file.txt');
-		writeFileSync(filePath, '');
-		expect(await isDir(filePath)).toBe(false);
-	});
-});

package/src/archive/filesystem/is-dir.ts

@@ -1,11 +0,0 @@
-import fsx from 'fs-extra';
-
-/**
- * Checks whether the given path points to a directory.
- * @param dirPath - The path to check.
- * @returns `true` if the path is a directory, `false` otherwise.
- */
-export async function isDir(dirPath: string) {
-	const stat = await fsx.stat(dirPath);
-	return stat.isDirectory();
-}

package/src/archive/filesystem/mkdir.spec.ts

@@ -1,37 +0,0 @@
-import { existsSync, rmSync, statSync } from 'node:fs';
-import { tmpdir } from 'node:os';
-import path from 'node:path';
-
-import { afterEach, describe, expect, it } from 'vitest';
-
-import { mkdir } from './mkdir.js';
-
-describe('mkdir', () => {
-	const testDir = path.join(tmpdir(), 'nitpicker-test-mkdir');
-
-	afterEach(() => {
-		rmSync(testDir, { recursive: true, force: true });
-	});
-
-	it('creates parent directory if it does not exist', () => {
-		const filePath = path.join(testDir, 'sub', 'file.txt');
-		mkdir(filePath);
-		expect(existsSync(path.join(testDir, 'sub'))).toBe(true);
-	});
-
-	it('does nothing if directory already exists', () => {
-		const filePath = path.join(testDir, 'sub', 'file.txt');
-		mkdir(filePath);
-		mkdir(filePath); // second call should not throw
-		expect(existsSync(path.join(testDir, 'sub'))).toBe(true);
-	});
-
-	it('creates directory with 0o700 permissions', () => {
-		const filePath = path.join(testDir, 'secure', 'file.txt');
-		mkdir(filePath);
-		const stats = statSync(path.join(testDir, 'secure'));
-
-		const mode = stats.mode & 0o777;
-		expect(mode).toBe(0o700);
-	});
-});

package/src/archive/filesystem/mkdir.ts

@@ -1,16 +0,0 @@
-import { existsSync, mkdirSync } from 'node:fs';
-import path from 'node:path';
-
-/**
- * Ensures the parent directory of the given file path exists.
- *
- * If the parent directory does not exist, it is created recursively
- * with permissions `0o700` (owner-only access).
- * @param filePath - The file path whose parent directory should be created.
- */
-export function mkdir(filePath: string) {
-	const { dir } = path.parse(filePath);
-	if (!existsSync(dir)) {
-		mkdirSync(path.resolve(dir), { recursive: true, mode: 0o700 });
-	}
-}

package/src/archive/filesystem/output-json.spec.ts

@@ -1,34 +0,0 @@
-import { readFileSync, rmSync, mkdirSync } from 'node:fs';
-import { tmpdir } from 'node:os';
-import path from 'node:path';
-
-import { afterEach, beforeEach, describe, expect, it } from 'vitest';
-
-import { outputJSON } from './output-json.js';
-
-describe('outputJSON', () => {
-	const testDir = path.join(tmpdir(), 'nitpicker-test-output-json');
-
-	beforeEach(() => {
-		mkdirSync(testDir, { recursive: true });
-	});
-
-	afterEach(() => {
-		rmSync(testDir, { recursive: true, force: true });
-	});
-
-	it('writes JSON data to file with 2-space indentation', async () => {
-		const filePath = path.join(testDir, 'data.json');
-		await outputJSON(filePath, { key: 'value' });
-		const content = readFileSync(filePath, 'utf8');
-		expect(JSON.parse(content)).toEqual({ key: 'value' });
-		expect(content).toContain('  '); // 2-space indent
-	});
-
-	it('creates parent directories if they do not exist', async () => {
-		const filePath = path.join(testDir, 'nested', 'dir', 'data.json');
-		await outputJSON(filePath, [1, 2, 3]);
-		const content = readFileSync(filePath, 'utf8');
-		expect(JSON.parse(content)).toEqual([1, 2, 3]);
-	});
-});

package/src/archive/filesystem/output-json.ts

@@ -1,16 +0,0 @@
-import { promises as fs } from 'node:fs';
-
-import { mkdir } from './mkdir.js';
-
-/**
- * Writes data to a JSON file at the specified path.
- *
- * Creates parent directories if they do not exist.
- * The output is formatted with 2-space indentation.
- * @param filePath - The absolute or relative path to the JSON file to write.
- * @param data - The data to serialize as JSON and write to the file.
- */
-export async function outputJSON(filePath: string, data: unknown) {
-	mkdir(filePath);
-	await fs.writeFile(filePath, JSON.stringify(data, null, 2), { encoding: 'utf8' });
-}

package/src/archive/filesystem/output-text.spec.ts

@@ -1,31 +0,0 @@
-import { readFileSync, mkdirSync, rmSync } from 'node:fs';
-import { tmpdir } from 'node:os';
-import path from 'node:path';
-
-import { afterEach, beforeEach, describe, expect, it } from 'vitest';
-
-import { outputText } from './output-text.js';
-
-describe('outputText', () => {
-	const testDir = path.join(tmpdir(), 'nitpicker-test-output-text');
-
-	beforeEach(() => {
-		mkdirSync(testDir, { recursive: true });
-	});
-
-	afterEach(() => {
-		rmSync(testDir, { recursive: true, force: true });
-	});
-
-	it('writes text to file', async () => {
-		const filePath = path.join(testDir, 'file.txt');
-		await outputText(filePath, 'hello world');
-		expect(readFileSync(filePath, 'utf8')).toBe('hello world');
-	});
-
-	it('creates parent directories if needed', async () => {
-		const filePath = path.join(testDir, 'sub', 'file.txt');
-		await outputText(filePath, 'nested');
-		expect(readFileSync(filePath, 'utf8')).toBe('nested');
-	});
-});

package/src/archive/filesystem/output-text.ts

@@ -1,35 +0,0 @@
-import { promises as fs } from 'node:fs';
-import path from 'node:path';
-
-import { mkdir } from './mkdir.js';
-
-let filePathTooLongCount = 0;
-
-/**
- * Writes text data to a file at the specified path.
- *
- * Creates parent directories if they do not exist.
- * If the file path exceeds the OS limit (ENAMETOOLONG), the file is saved
- * with an auto-generated short name and an accompanying `.meta.txt` file
- * that records the original file path.
- * @param filePath - The absolute or relative path to the text file to write.
- * @param data - The text content to write to the file.
- */
-export async function outputText(filePath: string, data: string) {
-	mkdir(filePath);
-	await fs.writeFile(filePath, data, { encoding: 'utf8' }).catch(async (error) => {
-		if (error instanceof Error && 'code' in error && error.code === 'ENAMETOOLONG') {
-			// eslint-disable-next-line no-console
-			console.error(`File path too long: ${filePath}`);
-			const dir = path.dirname(filePath);
-			const altFileName = `__file_path_too_long_${(filePathTooLongCount++).toString().padStart(4, '0')}`;
-			const ext = path.extname(filePath);
-			const altFilePath = path.resolve(dir, `${altFileName}${ext}`);
-			// eslint-disable-next-line no-console
-			console.error(`Try to save to: ${altFilePath}`);
-			const altMetaFilePath = path.resolve(dir, `${altFileName}.meta.txt`);
-			await outputText(altFilePath, data);
-			await outputText(altMetaFilePath, `Original file path: ${filePath}`);
-		}
-	});
-}

package/src/archive/filesystem/read-json.spec.ts

@@ -1,26 +0,0 @@
-import { writeFileSync, mkdirSync, rmSync } from 'node:fs';
-import { tmpdir } from 'node:os';
-import path from 'node:path';
-
-import { afterEach, beforeEach, describe, expect, it } from 'vitest';
-
-import { readJSON } from './read-json.js';
-
-describe('readJSON', () => {
-	const testDir = path.join(tmpdir(), 'nitpicker-test-read-json');
-
-	beforeEach(() => {
-		mkdirSync(testDir, { recursive: true });
-	});
-
-	afterEach(() => {
-		rmSync(testDir, { recursive: true, force: true });
-	});
-
-	it('reads and parses JSON from file', async () => {
-		const filePath = path.join(testDir, 'data.json');
-		writeFileSync(filePath, JSON.stringify({ foo: 'bar' }));
-		const result = await readJSON<{ foo: string }>(filePath);
-		expect(result).toEqual({ foo: 'bar' });
-	});
-});

package/src/archive/filesystem/read-json.ts

@@ -1,12 +0,0 @@
-import { promises as fs } from 'node:fs';
-
-/**
- * Reads and parses a JSON file from the specified path.
- * @template T - The expected type of the parsed JSON content. Defaults to `unknown`.
- * @param filePath - The absolute or relative path to the JSON file to read.
- * @returns The parsed JSON content, cast to the specified generic type.
- */
-export async function readJSON<T = unknown>(filePath: string) {
-	const data = await fs.readFile(filePath, { encoding: 'utf8' });
-	return JSON.parse(data) as T;
-}

package/src/archive/filesystem/read-text.spec.ts

@@ -1,25 +0,0 @@
-import { writeFileSync, mkdirSync, rmSync } from 'node:fs';
-import { tmpdir } from 'node:os';
-import path from 'node:path';
-
-import { afterEach, beforeEach, describe, expect, it } from 'vitest';
-
-import { readText } from './read-text.js';
-
-describe('readText', () => {
-	const testDir = path.join(tmpdir(), 'nitpicker-test-read-text');
-
-	beforeEach(() => {
-		mkdirSync(testDir, { recursive: true });
-	});
-
-	afterEach(() => {
-		rmSync(testDir, { recursive: true, force: true });
-	});
-
-	it('reads text content from file', async () => {
-		const filePath = path.join(testDir, 'file.txt');
-		writeFileSync(filePath, 'test content');
-		expect(await readText(filePath)).toBe('test content');
-	});
-});

package/src/archive/filesystem/read-text.ts

@@ -1,11 +0,0 @@
-import { promises as fs } from 'node:fs';
-
-/**
- * Reads the entire contents of a text file as a UTF-8 string.
- * @param filePath - The absolute or relative path to the text file to read.
- * @returns The text content of the file.
- */
-export async function readText(filePath: string) {
-	const data = await fs.readFile(filePath, { encoding: 'utf8' });
-	return data;
-}

package/src/archive/filesystem/readline.spec.ts

@@ -1,29 +0,0 @@
-import { writeFileSync, mkdirSync, rmSync } from 'node:fs';
-import { tmpdir } from 'node:os';
-import path from 'node:path';
-
-import { afterEach, beforeEach, describe, expect, it } from 'vitest';
-
-import { readline } from './readline.js';
-
-describe('readline', () => {
-	const testDir = path.join(tmpdir(), 'nitpicker-test-readline');
-
-	beforeEach(() => {
-		mkdirSync(testDir, { recursive: true });
-	});
-
-	afterEach(() => {
-		rmSync(testDir, { recursive: true, force: true });
-	});
-
-	it('invokes callback for each line', async () => {
-		const filePath = path.join(testDir, 'lines.txt');
-		writeFileSync(filePath, 'line1\nline2\nline3');
-		const lines: string[] = [];
-		await readline(filePath, (line) => {
-			lines.push(line);
-		});
-		expect(lines).toEqual(['line1', 'line2', 'line3']);
-	});
-});

package/src/archive/filesystem/readline.ts

@@ -1,30 +0,0 @@
-import { createReadStream } from 'node:fs';
-import Readline from 'node:readline';
-
-/**
- * Reads a file line by line and invokes the callback for each line.
- *
- * The callback may return a Promise for asynchronous processing.
- * All callback results are collected and awaited via `Promise.all` before returning.
- * @param filePath - The path to the file to read line by line.
- * @param callback - A function invoked for each line of the file.
- *   May return a Promise for asynchronous operations.
- * @returns A promise that resolves when all line callbacks have completed.
- */
-export async function readline(
-	filePath: string,
-	callback: (line: string) => Promise<void> | void,
-) {
-	const stream = createReadStream(filePath);
-	const rLine = Readline.createInterface(stream);
-	const promiseBuffer: (Promise<void> | void)[] = [];
-	await new Promise<void>((resolve) => {
-		rLine.on('line', (line) => {
-			promiseBuffer.push(callback(line));
-		});
-		rLine.on('close', () => {
-			resolve();
-		});
-	});
-	return Promise.all(promiseBuffer);
-}

package/src/archive/filesystem/remove.spec.ts

@@ -1,34 +0,0 @@
-import { writeFileSync, existsSync, mkdirSync, rmSync } from 'node:fs';
-import { tmpdir } from 'node:os';
-import path from 'node:path';
-
-import { afterEach, beforeEach, describe, expect, it } from 'vitest';
-
-import { remove } from './remove.js';
-
-describe('remove', () => {
-	const testDir = path.join(tmpdir(), 'nitpicker-test-remove');
-
-	beforeEach(() => {
-		mkdirSync(testDir, { recursive: true });
-	});
-
-	afterEach(() => {
-		rmSync(testDir, { recursive: true, force: true });
-	});
-
-	it('removes a file', async () => {
-		const filePath = path.join(testDir, 'file.txt');
-		writeFileSync(filePath, 'data');
-		await remove(filePath);
-		expect(existsSync(filePath)).toBe(false);
-	});
-
-	it('removes a directory recursively', async () => {
-		const subDir = path.join(testDir, 'sub');
-		mkdirSync(subDir);
-		writeFileSync(path.join(subDir, 'file.txt'), 'data');
-		await remove(subDir);
-		expect(existsSync(subDir)).toBe(false);
-	});
-});