@1-/scan 0.1.6 → 0.1.8
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +74 -131
- package/_.js +60 -27
- package/package.json +5 -3
- package/save.js +8 -8
- package/dirWalk.js +0 -35
- package/hash.js +0 -7
- package/sqlite.js +0 -7
- package/trans.js +0 -11
package/README.md
CHANGED
|
@@ -3,20 +3,19 @@
|
|
|
3
3
|
---
|
|
4
4
|
|
|
5
5
|
<a id="en"></a>
|
|
6
|
-
# @1-/scan :
|
|
6
|
+
# @1-/scan : SQLite-backed incremental directory scanner
|
|
7
7
|
|
|
8
|
-
Incrementally scans directory files, compares file sizes and modification times to detect changes, synchronizes metadata to SQLite database, and returns
|
|
8
|
+
Incrementally scans directory files, compares file sizes and modification times to detect changes, synchronizes metadata to SQLite database, and returns list of changed relative paths.
|
|
9
9
|
|
|
10
|
-
## Features
|
|
10
|
+
## 1. Features
|
|
11
11
|
|
|
12
|
-
- **Incremental
|
|
13
|
-
- **Key Optimization**: Stores
|
|
14
|
-
- **
|
|
15
|
-
- **Transactional Integrity**: Performs updates and deletions
|
|
16
|
-
- **
|
|
17
|
-
- **Native Database**: Integrates Bun native `bun:sqlite` module, eliminating external database driver dependencies.
|
|
12
|
+
- **Incremental Scan**: Compares size and modification time, filtering unchanged files to reduce disk I/O.
|
|
13
|
+
- **Key Length Optimization**: Stores raw bytes for paths up to 16 bytes. Converts longer paths into 16-byte MD5 hashes to optimize database index space and query performance.
|
|
14
|
+
- **Memory Optimization**: Uses BinMap and BinSet to store binary keys in memory, avoiding string decoding overhead and reducing memory footprint.
|
|
15
|
+
- **Transactional Integrity**: Performs metadata updates and deletions in database transactions to ensure consistency.
|
|
16
|
+
- **Zero Configuration**: Integrates @1-/sqlite to initialize database schema and manage database connections automatically.
|
|
18
17
|
|
|
19
|
-
## Usage
|
|
18
|
+
## 2. Usage
|
|
20
19
|
|
|
21
20
|
### Basic Incremental Scan
|
|
22
21
|
|
|
@@ -25,12 +24,13 @@ import scan from "@1-/scan";
|
|
|
25
24
|
|
|
26
25
|
const dir = "./data";
|
|
27
26
|
const db_path = "./scan_record.db";
|
|
27
|
+
const files = ["file1.txt", "file2.txt"];
|
|
28
28
|
|
|
29
|
-
// Scan
|
|
30
|
-
const [updated_paths, upsert] = await scan(dir, db_path);
|
|
29
|
+
// Scan file list, sync metadata to SQLite, return changed relative paths and upsert function
|
|
30
|
+
const [updated_paths, upsert] = await scan(dir, db_path, files);
|
|
31
31
|
|
|
32
|
-
//
|
|
33
|
-
using
|
|
32
|
+
// Close database automatically when exiting scope
|
|
33
|
+
using _ = upsert;
|
|
34
34
|
|
|
35
35
|
console.log("Updated files:", updated_paths);
|
|
36
36
|
|
|
@@ -40,35 +40,11 @@ for (const rel_path of updated_paths) {
|
|
|
40
40
|
}
|
|
41
41
|
```
|
|
42
42
|
|
|
43
|
-
###
|
|
44
|
-
|
|
45
|
-
```javascript
|
|
46
|
-
import scan from "@1-/scan";
|
|
47
|
-
|
|
48
|
-
const dir = "./data";
|
|
49
|
-
const db_path = "./scan_record.db";
|
|
50
|
-
|
|
51
|
-
// Ignore temporary files and specific configurations
|
|
52
|
-
const ignore = (kind, rel_path) => {
|
|
53
|
-
return rel_path.startsWith("temp/") || rel_path === "config.json";
|
|
54
|
-
};
|
|
55
|
-
|
|
56
|
-
const [updated_paths, upsert] = await scan(dir, db_path, ignore);
|
|
57
|
-
using _upsert = upsert;
|
|
58
|
-
|
|
59
|
-
console.log("Synced. Updated files:", updated_paths);
|
|
60
|
-
|
|
61
|
-
// Update scanned file metadata in database
|
|
62
|
-
for (const rel_path of updated_paths) {
|
|
63
|
-
await upsert(rel_path);
|
|
64
|
-
}
|
|
65
|
-
```
|
|
66
|
-
|
|
67
|
-
### Bulk Storage Module Usage
|
|
43
|
+
### Bulk Storage Module
|
|
68
44
|
|
|
69
45
|
```javascript
|
|
70
46
|
import save from "@1-/scan/save.js";
|
|
71
|
-
import sqlite from "@1-/
|
|
47
|
+
import sqlite from "@1-/sqlite";
|
|
72
48
|
|
|
73
49
|
const db = sqlite("./scan_record.db");
|
|
74
50
|
|
|
@@ -78,46 +54,41 @@ save(db, [["file.txt", new Uint8Array([1, 2, 3]), 123, 1620000000]], [new Uint8A
|
|
|
78
54
|
db.close();
|
|
79
55
|
```
|
|
80
56
|
|
|
81
|
-
## Design
|
|
57
|
+
## 3. Design
|
|
82
58
|
|
|
83
|
-
|
|
59
|
+
Main entry orchestrates modules to scan directories and synchronize metadata.
|
|
84
60
|
|
|
85
|
-

|
|
86
62
|
|
|
87
|
-
1. **Initialize Connection
|
|
88
|
-
2. **Load Records
|
|
89
|
-
3. **
|
|
90
|
-
4. **Delete
|
|
91
|
-
5. **Independent Sync Helper (`save.js`)**: Exported independent module to execute bulk updates and deletions in transactions.
|
|
63
|
+
1. **Initialize Connection**: Calls `@1-/sqlite` to open SQLite database.
|
|
64
|
+
2. **Load Records**: `load.js` checks `scanMtimeLen` table, creates it if missing. Reads stored hashes, sizes, and modification times to restore memory mappings.
|
|
65
|
+
3. **Compare Files**: Iterates over input file list. Maps paths to 16-byte binary keys via `@1-/hash`. Adds files with mismatched size or modification time to update list.
|
|
66
|
+
4. **Delete and Return**: Deletes absent or unscanned records in transaction. Returns changed paths list and `upsert` function for persistence.
|
|
92
67
|
|
|
93
|
-
## Tech Stack
|
|
68
|
+
## 4. Tech Stack
|
|
94
69
|
|
|
95
70
|
- **Bun**: Runtime environment and test framework.
|
|
96
|
-
-
|
|
97
|
-
- **@1-/
|
|
71
|
+
- **@1-/sqlite**: Database connection management and transaction wrapper.
|
|
72
|
+
- **@1-/hash**: Length-bounded MD5 hash utility.
|
|
98
73
|
- **@3-/vb**: Variable-length byte (Varint) encoder and decoder.
|
|
99
|
-
- **@3-/binmap / @3-/binset**:
|
|
74
|
+
- **@3-/binmap / @3-/binset**: Rust and WebAssembly binary key containers.
|
|
100
75
|
|
|
101
|
-
## Code Structure
|
|
76
|
+
## 5. Code Structure
|
|
102
77
|
|
|
103
|
-
```
|
|
78
|
+
```text
|
|
104
79
|
.
|
|
105
80
|
├── src
|
|
106
|
-
│ ├── _.js #
|
|
107
|
-
│ ├──
|
|
108
|
-
│
|
|
109
|
-
|
|
110
|
-
│ ├── save.js # Independent helper executing bulk updates and deletions
|
|
111
|
-
│ ├── sqlite.js # Connection manager instantiating SQLite database
|
|
112
|
-
│ └── trans.js # Transaction wrapper providing rollback mechanism
|
|
113
|
-
└── tests # Test directory
|
|
81
|
+
│ ├── _.js # Core flow controller
|
|
82
|
+
│ ├── load.js # Table schema initialization and loader
|
|
83
|
+
│ └── save.js # Bulk update and delete helper
|
|
84
|
+
└── tests # Unit tests
|
|
114
85
|
```
|
|
115
86
|
|
|
116
|
-
## History
|
|
87
|
+
## 6. History
|
|
117
88
|
|
|
118
|
-
SQLite was created by D. Richard Hipp in 2000 while designing board software for
|
|
89
|
+
SQLite was created by D. Richard Hipp in 2000 while designing board software for guided-missile destroyers. The system originally depended on commercial database that required constant database administration; connection loss could stall the entire damage control application. Hipp designed serverless, zero-configuration embedded database that directly reads and writes local files, marking the birth of SQLite.
|
|
119
90
|
|
|
120
|
-
To conserve
|
|
91
|
+
To conserve space and reduce latency, SQLite utilizes Varint (variable-length integer) encoding for metadata storage. Under this scheme, small integers consume only 1 byte, while larger numbers scale dynamically. This library inherits that design philosophy, compressing file metadata into varints for memory storage to ensure minimal footprint and high synchronization performance.
|
|
121
92
|
## About
|
|
122
93
|
|
|
123
94
|
This library is developed by [WebC.site](https://webc.site).
|
|
@@ -127,20 +98,19 @@ This library is developed by [WebC.site](https://webc.site).
|
|
|
127
98
|
---
|
|
128
99
|
|
|
129
100
|
<a id="zh"></a>
|
|
130
|
-
# @1-/scan :
|
|
101
|
+
# @1-/scan : 基于 SQLite 的目录增量扫描工具
|
|
131
102
|
|
|
132
|
-
|
|
103
|
+
增量扫描目录文件,比对文件大小与修改时间检测变更,同步元数据至 SQLite 数据库,返回已变更相对路径列表。
|
|
133
104
|
|
|
134
|
-
## 功能介绍
|
|
105
|
+
## 1. 功能介绍
|
|
135
106
|
|
|
136
|
-
-
|
|
137
|
-
-
|
|
138
|
-
-
|
|
139
|
-
-
|
|
140
|
-
-
|
|
141
|
-
- **原生依赖**:基于 Bun 内置 `bun:sqlite` 模块,无需额外安装或编译数据库驱动。
|
|
107
|
+
- **增量扫描**:比对大小与修改时间,过滤未变更文件,减少磁盘读写。
|
|
108
|
+
- **键长优化**:路径长度不大于 16 字节存储原始字节,超出 16 字节转换为 16 字节 MD5 值,优化索引空间与查询性能。
|
|
109
|
+
- **内存优化**:使用 BinMap 与 BinSet 存储二进制键,避免字符串解码,降低内存占用。
|
|
110
|
+
- **事务保障**:元数据变更与删除操作合并在数据库事务中执行,确保数据一致性。
|
|
111
|
+
- **零配置**:集成 @1-/sqlite,自动初始化数据库表结构,管理数据库连接。
|
|
142
112
|
|
|
143
|
-
## 使用演示
|
|
113
|
+
## 2. 使用演示
|
|
144
114
|
|
|
145
115
|
### 基础增量扫描
|
|
146
116
|
|
|
@@ -149,49 +119,27 @@ import scan from "@1-/scan";
|
|
|
149
119
|
|
|
150
120
|
const dir = "./data";
|
|
151
121
|
const db_path = "./scan_record.db";
|
|
122
|
+
const files = ["file1.txt", "file2.txt"];
|
|
152
123
|
|
|
153
|
-
//
|
|
154
|
-
const [updated_paths, upsert] = await scan(dir, db_path);
|
|
124
|
+
// 扫描文件列表并同步至 SQLite,返回已变更的相对路径列表与更新函数
|
|
125
|
+
const [updated_paths, upsert] = await scan(dir, db_path, files);
|
|
155
126
|
|
|
156
|
-
//
|
|
157
|
-
using
|
|
127
|
+
// 退出作用域自动关闭数据库
|
|
128
|
+
using _ = upsert;
|
|
158
129
|
|
|
159
130
|
console.log("更新文件列表:", updated_paths);
|
|
160
131
|
|
|
161
|
-
//
|
|
132
|
+
// 更新已处理文件的元数据至数据库
|
|
162
133
|
for (const rel_path of updated_paths) {
|
|
163
134
|
await upsert(rel_path);
|
|
164
135
|
}
|
|
165
136
|
```
|
|
166
137
|
|
|
167
|
-
###
|
|
168
|
-
|
|
169
|
-
```javascript
|
|
170
|
-
import scan from "@1-/scan";
|
|
171
|
-
|
|
172
|
-
const dir = "./data";
|
|
173
|
-
const db_path = "./scan_record.db";
|
|
174
|
-
|
|
175
|
-
// 过滤临时文件与特定配置
|
|
176
|
-
const ignore = (kind, rel_path) => {
|
|
177
|
-
return rel_path.startsWith("temp/") || rel_path === "config.json";
|
|
178
|
-
};
|
|
179
|
-
|
|
180
|
-
const [updated_paths, upsert] = await scan(dir, db_path, ignore);
|
|
181
|
-
using _upsert = upsert;
|
|
182
|
-
|
|
183
|
-
console.log("已同步,更新列表:", updated_paths);
|
|
184
|
-
|
|
185
|
-
for (const rel_path of updated_paths) {
|
|
186
|
-
await upsert(rel_path);
|
|
187
|
-
}
|
|
188
|
-
```
|
|
189
|
-
|
|
190
|
-
### 批量存储模块使用
|
|
138
|
+
### 独立批量存储模块
|
|
191
139
|
|
|
192
140
|
```javascript
|
|
193
141
|
import save from "@1-/scan/save.js";
|
|
194
|
-
import sqlite from "@1-/
|
|
142
|
+
import sqlite from "@1-/sqlite";
|
|
195
143
|
|
|
196
144
|
const db = sqlite("./scan_record.db");
|
|
197
145
|
|
|
@@ -201,46 +149,41 @@ save(db, [["file.txt", new Uint8Array([1, 2, 3]), 123, 1620000000]], [new Uint8A
|
|
|
201
149
|
db.close();
|
|
202
150
|
```
|
|
203
151
|
|
|
204
|
-
## 设计思路
|
|
152
|
+
## 3. 设计思路
|
|
205
153
|
|
|
206
|
-
|
|
154
|
+
主入口调度各模块,协作完成目录扫描与数据同步。
|
|
207
155
|
|
|
208
|
-

|
|
209
157
|
|
|
210
|
-
1.
|
|
211
|
-
2.
|
|
212
|
-
3.
|
|
213
|
-
4.
|
|
214
|
-
5. **独立批量存储模块 (`save.js`)**:供外部调用的独立工具模块,用于在事务中批量写入与删除。
|
|
158
|
+
1. **初始化连接**:调用 `@1-/sqlite` 打开 SQLite 数据库。
|
|
159
|
+
2. **加载记录**:`load.js` 检查 `scanMtimeLen` 表,表不存在则自动创建。读取已记录哈希、大小及修改时间,恢复内存映射。
|
|
160
|
+
3. **比对文件**:遍历输入文件列表,利用 `@1-/hash` 将路径映射为 16 字节二进制键。若大小或修改时间不一致,则加入变更列表。
|
|
161
|
+
4. **删除与返回**:在事务中批量删除已被物理移除或不再扫描记录。返回变更路径列表与 `upsert` 函数,供外部持久化。
|
|
215
162
|
|
|
216
|
-
## 技术栈
|
|
163
|
+
## 4. 技术栈
|
|
217
164
|
|
|
218
165
|
- **Bun**:JavaScript 运行时与测试框架。
|
|
219
|
-
-
|
|
220
|
-
- **@1-/
|
|
221
|
-
- **@3-/vb**:Varint
|
|
222
|
-
- **@3-/binmap / @3-/binset
|
|
166
|
+
- **@1-/sqlite**:SQLite 连接管理与事务封装。
|
|
167
|
+
- **@1-/hash**:长度限制的 MD5 哈希工具。
|
|
168
|
+
- **@3-/vb**:Varint 变长整型编码与解码器。
|
|
169
|
+
- **@3-/binmap / @3-/binset**:基于 Rust 与 WebAssembly 的高效二进制键容器。
|
|
223
170
|
|
|
224
|
-
##
|
|
171
|
+
## 5. Code Structure
|
|
225
172
|
|
|
226
|
-
```
|
|
173
|
+
```text
|
|
227
174
|
.
|
|
228
175
|
├── src
|
|
229
|
-
│ ├── _.js #
|
|
230
|
-
│ ├──
|
|
231
|
-
│
|
|
232
|
-
|
|
233
|
-
│ ├── save.js # 独立导出的批量持久化与删除辅助函数
|
|
234
|
-
│ ├── sqlite.js # 创建并配置 SQLite 数据库实例
|
|
235
|
-
│ └── trans.js # 封装 SQLite 事务,提供异常回滚机制
|
|
236
|
-
└── tests # 单元测试目录
|
|
176
|
+
│ ├── _.js # 核心控制流程
|
|
177
|
+
│ ├── load.js # 元数据表初始化与加载
|
|
178
|
+
│ └── save.js # 批量更新与删除工具
|
|
179
|
+
└── tests # 单元测试
|
|
237
180
|
```
|
|
238
181
|
|
|
239
|
-
## 历史故事
|
|
182
|
+
## 6. 历史故事
|
|
240
183
|
|
|
241
|
-
SQLite
|
|
184
|
+
SQLite 的诞生源自导弹驱逐舰板载损害控制软件项目。2000 年,D. Richard Hipp 为美国海军设计该系统时,遭遇商业数据库因配置复杂、无法承受断连和崩溃之痛点。Hipp 随后设计出免服务器配置、直接读写本地文件之嵌入式数据库,即 SQLite。
|
|
242
185
|
|
|
243
|
-
|
|
186
|
+
为节省空间与降低读写延迟,SQLite 广泛应用了 Varint(可变字节整型)编码。在这种编码下,小整数仅占用 1 字节,只有大数值才占用更多字节。本项目在内存存储设计中对文件大小与修改时间采用同样的压缩设计,契合 SQLite 节省空间与高效之设计哲学。
|
|
244
187
|
## 关于
|
|
245
188
|
|
|
246
189
|
本库由 [WebC.site](https://webc.site) 开发。
|
package/_.js
CHANGED
|
@@ -1,41 +1,74 @@
|
|
|
1
|
+
import sqlite from "@1-/sqlite";
|
|
2
|
+
import tx from "@1-/sqlite/tx.js";
|
|
1
3
|
import { BinMap } from "@3-/binmap";
|
|
2
4
|
import vbE from "@3-/vb/vbE.js";
|
|
3
|
-
import sqlite from "./sqlite.js";
|
|
4
5
|
import load from "./load.js";
|
|
5
|
-
import
|
|
6
|
-
import { stat } from "node:fs/promises";
|
|
6
|
+
import { stat as fsStat } from "node:fs/promises";
|
|
7
7
|
import { join } from "node:path";
|
|
8
8
|
import int from "@3-/int";
|
|
9
|
-
import
|
|
10
|
-
import
|
|
9
|
+
import strmd5 from "@1-/hash/strmd5.js";
|
|
10
|
+
import { BinSet } from "@3-/binset";
|
|
11
|
+
import u8eq from "@3-/u8/u8eq.js";
|
|
12
|
+
import { availableParallelism } from "node:os";
|
|
13
|
+
import pLimit from "@3-/plimit";
|
|
11
14
|
|
|
12
|
-
|
|
15
|
+
const stat = async (dir, rel_path) => {
|
|
16
|
+
const { size, mtimeMs: mtime_ms } = await fsStat(join(dir, rel_path));
|
|
17
|
+
return [size, int(mtime_ms), strmd5(rel_path)];
|
|
18
|
+
},
|
|
19
|
+
scanFiles = async (dir, files, existing, limit) => {
|
|
20
|
+
const scanned = new BinSet(),
|
|
21
|
+
update = [];
|
|
22
|
+
await Promise.all(
|
|
23
|
+
files.map((rel_path) =>
|
|
24
|
+
limit(async () => {
|
|
25
|
+
try {
|
|
26
|
+
const [size, mtime, hash] = await stat(dir, rel_path),
|
|
27
|
+
val = existing.get(hash);
|
|
28
|
+
|
|
29
|
+
scanned.add(hash);
|
|
30
|
+
|
|
31
|
+
if (!val || !u8eq(val, vbE([size, mtime]))) {
|
|
32
|
+
update.push(rel_path);
|
|
33
|
+
}
|
|
34
|
+
} catch {}
|
|
35
|
+
}),
|
|
36
|
+
),
|
|
37
|
+
);
|
|
38
|
+
return [scanned, update];
|
|
39
|
+
},
|
|
40
|
+
rmHashes = (db, rm) => {
|
|
41
|
+
if (rm.length > 0) {
|
|
42
|
+
tx(db, () => {
|
|
43
|
+
const del = db.prepare("DELETE FROM scanMtimeLen WHERE hash=?");
|
|
44
|
+
rm.forEach((hash) => del.run(hash));
|
|
45
|
+
});
|
|
46
|
+
}
|
|
47
|
+
},
|
|
48
|
+
newUpsert = (db, dir) => {
|
|
49
|
+
const insert = db.prepare("INSERT OR REPLACE INTO scanMtimeLen(hash,size,mtime)VALUES(?,?,?)"),
|
|
50
|
+
upsert = async (rel_path) => {
|
|
51
|
+
try {
|
|
52
|
+
const [size, mtime, hash] = await stat(dir, rel_path);
|
|
53
|
+
insert.run(hash, size, mtime);
|
|
54
|
+
} catch {}
|
|
55
|
+
};
|
|
56
|
+
upsert[Symbol.dispose] = () => db.close();
|
|
57
|
+
return upsert;
|
|
58
|
+
};
|
|
59
|
+
|
|
60
|
+
export default async (dir, db_path, files) => {
|
|
13
61
|
const db = sqlite(db_path),
|
|
14
62
|
existing = new BinMap(),
|
|
15
|
-
db_rows = load(db)
|
|
63
|
+
db_rows = load(db),
|
|
64
|
+
limit = pLimit(availableParallelism());
|
|
16
65
|
|
|
17
66
|
db_rows.forEach(({ hash, size, mtime }) => existing.set(hash, vbE([size, mtime])));
|
|
18
67
|
|
|
19
|
-
const [scanned,
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
if (to_delete.length > 0) {
|
|
23
|
-
trans(db, () => {
|
|
24
|
-
const del = db.prepare("DELETE FROM scanMtimeLen WHERE hash=?");
|
|
25
|
-
to_delete.forEach((h) => del.run(h));
|
|
26
|
-
});
|
|
27
|
-
}
|
|
28
|
-
|
|
29
|
-
const insert = db.prepare("INSERT OR REPLACE INTO scanMtimeLen(hash,size,mtime)VALUES(?,?,?)"),
|
|
30
|
-
upsert = async (rel_path) => {
|
|
31
|
-
const fp = join(dir, rel_path),
|
|
32
|
-
{ size, mtimeMs } = await stat(fp),
|
|
33
|
-
mtime = int(mtimeMs),
|
|
34
|
-
h = hash(rel_path);
|
|
35
|
-
insert.run(h, size, mtime);
|
|
36
|
-
};
|
|
68
|
+
const [scanned, update] = await scanFiles(dir, files, existing, limit),
|
|
69
|
+
rm = db_rows.filter(({ hash }) => !scanned.has(hash)).map(({ hash }) => hash);
|
|
37
70
|
|
|
38
|
-
|
|
71
|
+
rmHashes(db, rm);
|
|
39
72
|
|
|
40
|
-
return [
|
|
73
|
+
return [update, newUpsert(db, dir)];
|
|
41
74
|
};
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@1-/scan",
|
|
3
|
-
"version": "0.1.
|
|
3
|
+
"version": "0.1.8",
|
|
4
4
|
"description": "Incrementally scan directory files and track metadata in SQLite / 增量扫描目录文件并使用 SQLite 记录元数据",
|
|
5
5
|
"keywords": [
|
|
6
6
|
"directory",
|
|
@@ -21,11 +21,13 @@
|
|
|
21
21
|
".": "./_.js",
|
|
22
22
|
"./*": "./*"
|
|
23
23
|
},
|
|
24
|
-
"
|
|
25
|
-
"@1-/
|
|
24
|
+
"peerDependencies": {
|
|
25
|
+
"@1-/hash": "^0.1.0",
|
|
26
|
+
"@1-/sqlite": "^0.1.1",
|
|
26
27
|
"@3-/binmap": "^0.1.20",
|
|
27
28
|
"@3-/binset": "^0.1.6",
|
|
28
29
|
"@3-/int": "^0.1.1",
|
|
30
|
+
"@3-/plimit": "^0.1.3",
|
|
29
31
|
"@3-/u8": "^0.1.2",
|
|
30
32
|
"@3-/utf8": "^0.1.1",
|
|
31
33
|
"@3-/vb": "^0.1.6"
|
package/save.js
CHANGED
|
@@ -1,17 +1,17 @@
|
|
|
1
|
-
import
|
|
1
|
+
import tx from "@1-/sqlite/tx.js";
|
|
2
2
|
|
|
3
|
-
export default (db,
|
|
4
|
-
if (
|
|
5
|
-
|
|
6
|
-
if (
|
|
3
|
+
export default (db, update, rm) => {
|
|
4
|
+
if (update.length > 0 || rm.length > 0) {
|
|
5
|
+
tx(db, () => {
|
|
6
|
+
if (update.length > 0) {
|
|
7
7
|
const insert = db.prepare(
|
|
8
8
|
"INSERT OR REPLACE INTO scanMtimeLen(hash,size,mtime)VALUES(?,?,?)",
|
|
9
9
|
);
|
|
10
|
-
|
|
10
|
+
update.forEach(([_, h, size, mtime]) => insert.run(h, size, mtime));
|
|
11
11
|
}
|
|
12
|
-
if (
|
|
12
|
+
if (rm.length > 0) {
|
|
13
13
|
const del = db.prepare("DELETE FROM scanMtimeLen WHERE hash=?");
|
|
14
|
-
|
|
14
|
+
rm.forEach((h) => del.run(h));
|
|
15
15
|
}
|
|
16
16
|
});
|
|
17
17
|
}
|
package/dirWalk.js
DELETED
|
@@ -1,35 +0,0 @@
|
|
|
1
|
-
import { stat } from "node:fs/promises";
|
|
2
|
-
import { join } from "node:path";
|
|
3
|
-
import { FILE } from "@1-/walk";
|
|
4
|
-
import walkRelIgnore from "@1-/walk/walkRelIgnore.js";
|
|
5
|
-
import { BinSet } from "@3-/binset";
|
|
6
|
-
import u8eq from "@3-/u8/u8eq.js";
|
|
7
|
-
import vbE from "@3-/vb/vbE.js";
|
|
8
|
-
import int from "@3-/int";
|
|
9
|
-
import hash from "./hash.js";
|
|
10
|
-
|
|
11
|
-
export default async (dir, existing, ignore) => {
|
|
12
|
-
const scanned = new BinSet(),
|
|
13
|
-
to_update = [];
|
|
14
|
-
|
|
15
|
-
await walkRelIgnore(dir, async (kind, rel_path) => {
|
|
16
|
-
if (ignore && ignore(kind, rel_path) === false) {
|
|
17
|
-
return false;
|
|
18
|
-
}
|
|
19
|
-
if (kind === FILE) {
|
|
20
|
-
const { size, mtimeMs } = await stat(join(dir, rel_path)),
|
|
21
|
-
mtime = int(mtimeMs),
|
|
22
|
-
h = hash(rel_path);
|
|
23
|
-
|
|
24
|
-
scanned.add(h);
|
|
25
|
-
|
|
26
|
-
const val = existing.get(h);
|
|
27
|
-
if (val && u8eq(val, vbE([size, mtime]))) {
|
|
28
|
-
return;
|
|
29
|
-
}
|
|
30
|
-
to_update.push([rel_path, h, size, mtime]);
|
|
31
|
-
}
|
|
32
|
-
});
|
|
33
|
-
|
|
34
|
-
return [scanned, to_update];
|
|
35
|
-
};
|
package/hash.js
DELETED
package/sqlite.js
DELETED