@1-/scan 0.1.5 → 0.1.7
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +79 -161
- package/_.js +39 -21
- package/package.json +8 -7
- package/save.js +8 -8
- package/dirWalk.js +0 -35
- package/hash.js +0 -7
- package/sqlite.js +0 -7
- package/trans.js +0 -11
package/README.md
CHANGED
|
@@ -7,16 +7,15 @@
|
|
|
7
7
|
|
|
8
8
|
Incrementally scans directory files, compares file sizes and modification times to detect changes, synchronizes metadata to SQLite database, and returns updated relative paths.
|
|
9
9
|
|
|
10
|
-
## Features
|
|
10
|
+
## 1. Features
|
|
11
11
|
|
|
12
|
-
- **Incremental Scanning**:
|
|
12
|
+
- **Incremental Scanning**: Compares file sizes and modification times to process only new, modified, or deleted files, avoiding redundant read/write operations.
|
|
13
13
|
- **Key Optimization**: Stores relative paths within 16 bytes directly as raw bytes; hashes longer paths to 16-byte MD5 digests to optimize database index space and query performance.
|
|
14
|
-
- **
|
|
15
|
-
- **Transactional Integrity**:
|
|
16
|
-
- **
|
|
17
|
-
- **Native Database**: Integrates Bun native `bun:sqlite` module, eliminating external database driver dependencies.
|
|
14
|
+
- **Memory Efficiency**: Utilizes BinMap and BinSet to store binary keys in memory, avoiding string decoding overhead and reducing memory footprint.
|
|
15
|
+
- **Transactional Integrity**: Performs updates and deletions within database transactions to guarantee consistency.
|
|
16
|
+
- **Configuration-free**: Built-in automatic table initialization and connection management via @1-/sqlite.
|
|
18
17
|
|
|
19
|
-
## Usage
|
|
18
|
+
## 2. Usage
|
|
20
19
|
|
|
21
20
|
### Basic Incremental Scan
|
|
22
21
|
|
|
@@ -25,12 +24,13 @@ import scan from "@1-/scan";
|
|
|
25
24
|
|
|
26
25
|
const dir = "./data";
|
|
27
26
|
const db_path = "./scan_record.db";
|
|
27
|
+
const files = ["file1.txt", "file2.txt"];
|
|
28
28
|
|
|
29
|
-
// Scan
|
|
30
|
-
const [updated_paths, upsert] = await scan(dir, db_path);
|
|
29
|
+
// Scan file list and sync metadata to SQLite, returning modified relative paths and upsert function
|
|
30
|
+
const [updated_paths, upsert] = await scan(dir, db_path, files);
|
|
31
31
|
|
|
32
32
|
// Auto-close database when exiting scope
|
|
33
|
-
using
|
|
33
|
+
using _ = upsert;
|
|
34
34
|
|
|
35
35
|
console.log("Updated files:", updated_paths);
|
|
36
36
|
|
|
@@ -40,119 +40,77 @@ for (const rel_path of updated_paths) {
|
|
|
40
40
|
}
|
|
41
41
|
```
|
|
42
42
|
|
|
43
|
-
###
|
|
44
|
-
|
|
45
|
-
```javascript
|
|
46
|
-
import scan from "@1-/scan";
|
|
47
|
-
|
|
48
|
-
const dir = "./data";
|
|
49
|
-
const db_path = "./scan_record.db";
|
|
50
|
-
|
|
51
|
-
// Ignore temporary files and specific configurations
|
|
52
|
-
const ignore = (kind, rel_path) => {
|
|
53
|
-
return rel_path.startsWith("temp/") || rel_path === "config.json";
|
|
54
|
-
};
|
|
55
|
-
|
|
56
|
-
const [updated_paths, upsert] = await scan(dir, db_path, ignore);
|
|
57
|
-
using _upsert = upsert;
|
|
58
|
-
|
|
59
|
-
console.log("Synced. Updated files:", updated_paths);
|
|
60
|
-
|
|
61
|
-
for (const rel_path of updated_paths) {
|
|
62
|
-
await upsert(rel_path);
|
|
63
|
-
}
|
|
64
|
-
```
|
|
65
|
-
|
|
66
|
-
### Bulk Storage Module Usage
|
|
43
|
+
### Bulk Storage Module
|
|
67
44
|
|
|
68
45
|
```javascript
|
|
69
46
|
import save from "@1-/scan/save.js";
|
|
70
|
-
import sqlite from "@1-/
|
|
47
|
+
import sqlite from "@1-/sqlite";
|
|
71
48
|
|
|
72
49
|
const db = sqlite("./scan_record.db");
|
|
73
50
|
|
|
74
51
|
// Bulk update and delete metadata
|
|
75
|
-
save(
|
|
76
|
-
db,
|
|
77
|
-
[
|
|
78
|
-
["file.txt", new Uint8Array([1, 2, 3]), 123, 1620000000]
|
|
79
|
-
],
|
|
80
|
-
[
|
|
81
|
-
new Uint8Array([4, 5, 6])
|
|
82
|
-
]
|
|
83
|
-
);
|
|
52
|
+
save(db, [["file.txt", new Uint8Array([1, 2, 3]), 123, 1620000000]], [new Uint8Array([4, 5, 6])]);
|
|
84
53
|
|
|
85
54
|
db.close();
|
|
86
55
|
```
|
|
87
56
|
|
|
88
|
-
## Design Ideas
|
|
57
|
+
## 3. Design Ideas
|
|
89
58
|
|
|
90
|
-
The
|
|
59
|
+
The entry point orchestrates independent modules to execute the incremental scanning and synchronization flow.
|
|
91
60
|
|
|
92
|
-
|
|
93
|
-
graph TD
|
|
94
|
-
Entry["_.js (Entry Point)"] -->|1. Initialize Connection| Sqlite["sqlite.js"]
|
|
95
|
-
Entry -->|2. Load Existing Records| Load["load.js"]
|
|
96
|
-
Entry -->|3. Walk & Compare Files| DirWalk["dirWalk.js"]
|
|
97
|
-
DirWalk -->|Invoke| Walk["@1-/walk/walkRelIgnore"]
|
|
98
|
-
DirWalk -->|Process Path Keys| Hash["hash.js"]
|
|
99
|
-
Entry -->|4. Delete Absent & Return Upsert| Trans["trans.js"]
|
|
100
|
-
Save["save.js (Independent Sync Helper)"] -->|Transaction Wrapper| Trans
|
|
101
|
-
```
|
|
61
|
+

|
|
102
62
|
|
|
103
|
-
1. **Initialize Connection
|
|
104
|
-
2. **Load Records
|
|
105
|
-
3. **
|
|
106
|
-
4. **Delete
|
|
107
|
-
5. **Independent Sync Helper (`save.js`)**: Exported independent module to execute bulk updates and deletions in transactions.
|
|
63
|
+
1. **Initialize Connection**: Invokes `@1-/sqlite` to open the SQLite database.
|
|
64
|
+
2. **Load Records**: `load.js` checks if the `scanMtimeLen` table exists and creates it if missing. It retrieves stored hashes, sizes, and modification times, and reconstructs the memory mapping.
|
|
65
|
+
3. **Compare Metadata**: Iterates over the input file list, mapping paths to 16-byte binary keys via `@1-/hash`. It adds files with size or modification time mismatches to the update list.
|
|
66
|
+
4. **Delete and Return**: Deletes absent records in a transaction and returns the changed path list along with the `upsert` function for external persistence.
|
|
108
67
|
|
|
109
|
-
## Tech Stack
|
|
68
|
+
## 4. Tech Stack
|
|
110
69
|
|
|
111
70
|
- **Bun**: Runtime environment and test framework.
|
|
112
|
-
-
|
|
113
|
-
- **@1-/
|
|
71
|
+
- **@1-/sqlite**: Database connection management and transaction wrapper.
|
|
72
|
+
- **@1-/hash**: Length-bounded MD5 hash utility.
|
|
114
73
|
- **@3-/vb**: Variable-length byte (Varint) encoder and decoder.
|
|
115
|
-
- **@3-/binmap / @3-/binset**:
|
|
74
|
+
- **@3-/binmap / @3-/binset**: Rust and WebAssembly binary key containers.
|
|
116
75
|
|
|
117
|
-
##
|
|
76
|
+
## 5. Code Structure
|
|
118
77
|
|
|
119
|
-
```
|
|
78
|
+
```text
|
|
120
79
|
.
|
|
121
80
|
├── src
|
|
122
|
-
│ ├── _.js #
|
|
123
|
-
│ ├──
|
|
124
|
-
│
|
|
125
|
-
|
|
126
|
-
│ ├── save.js # Independent helper executing bulk updates and deletions
|
|
127
|
-
│ ├── sqlite.js # Connection manager instantiating SQLite database
|
|
128
|
-
│ └── trans.js # Transaction wrapper providing rollback mechanism
|
|
129
|
-
└── tests # Test directory
|
|
81
|
+
│ ├── _.js # Core flow controller
|
|
82
|
+
│ ├── load.js # Table schema initialization and loader
|
|
83
|
+
│ └── save.js # Bulk update and delete helper
|
|
84
|
+
└── tests # Unit tests
|
|
130
85
|
```
|
|
131
86
|
|
|
132
|
-
## History
|
|
87
|
+
## 6. History
|
|
88
|
+
|
|
89
|
+
SQLite was created by D. Richard Hipp in 2000 while designing board software for guided-missile destroyers. The system originally depended on a commercial database that required constant database administration; a connection loss could stall the entire damage control application. Hipp designed a serverless, zero-configuration embedded database that directly reads and writes local files, marking the birth of SQLite.
|
|
90
|
+
|
|
91
|
+
To conserve space and reduce latency, SQLite utilizes Varint (variable-length integer) encoding for metadata storage. Under this scheme, small integers consume only 1 byte, while larger numbers scale dynamically. This library inherits that design philosophy, compressing file metadata into varints for memory storage to ensure minimal footprint and high synchronization performance.
|
|
92
|
+
## About
|
|
133
93
|
|
|
134
|
-
|
|
94
|
+
This library is developed by [WebC.site](https://webc.site).
|
|
135
95
|
|
|
136
|
-
|
|
137
|
-
../doc/en/about.md
|
|
96
|
+
[WebC.site](https://webc.site): A new paradigm of web development for AI
|
|
138
97
|
|
|
139
98
|
---
|
|
140
99
|
|
|
141
100
|
<a id="zh"></a>
|
|
142
101
|
# @1-/scan : 增量扫描目录文件并使用 SQLite 记录元数据
|
|
143
102
|
|
|
144
|
-
|
|
103
|
+
增量扫描目录文件,比对大小与修改时间检测变更,同步元数据至 SQLite 数据库,返回发生变更的相对路径列表。
|
|
145
104
|
|
|
146
|
-
## 功能介绍
|
|
105
|
+
## 1. 功能介绍
|
|
147
106
|
|
|
148
|
-
-
|
|
149
|
-
-
|
|
150
|
-
-
|
|
151
|
-
-
|
|
152
|
-
-
|
|
153
|
-
- **原生依赖**:基于 Bun 内置 `bun:sqlite` 模块,免去安装与编译数据库驱动步骤。
|
|
107
|
+
- **增量扫描**:比对大小与修改时间,过滤未变更文件,减少磁盘读写。
|
|
108
|
+
- **路径键优化**:路径长度不大于 16 字节时存储原始字节,超出 16 字节则转换为 16 字节 MD5 值,优化索引空间与查询性能。
|
|
109
|
+
- **内存优化**:在内存中使用 BinMap 与 BinSet 存储二进制哈希键,避免字符串解码开销,降低内存占用。
|
|
110
|
+
- **事务保障**:变更及删除操作合并在数据库事务中执行,确保数据一致性。
|
|
111
|
+
- **免除配置**:基于 @1-/sqlite,内置表初始化与连接管理,开箱即用。
|
|
154
112
|
|
|
155
|
-
## 使用演示
|
|
113
|
+
## 2. 使用演示
|
|
156
114
|
|
|
157
115
|
### 基础增量扫描
|
|
158
116
|
|
|
@@ -161,12 +119,13 @@ import scan from "@1-/scan";
|
|
|
161
119
|
|
|
162
120
|
const dir = "./data";
|
|
163
121
|
const db_path = "./scan_record.db";
|
|
122
|
+
const files = ["file1.txt", "file2.txt"];
|
|
164
123
|
|
|
165
|
-
//
|
|
166
|
-
const [updated_paths, upsert] = await scan(dir, db_path);
|
|
124
|
+
// 扫描文件列表并同步至 SQLite,返回发生变更的相对路径列表与更新函数
|
|
125
|
+
const [updated_paths, upsert] = await scan(dir, db_path, files);
|
|
167
126
|
|
|
168
127
|
// 退出作用域时自动关闭数据库
|
|
169
|
-
using
|
|
128
|
+
using _ = upsert;
|
|
170
129
|
|
|
171
130
|
console.log("更新文件列表:", updated_paths);
|
|
172
131
|
|
|
@@ -176,98 +135,57 @@ for (const rel_path of updated_paths) {
|
|
|
176
135
|
}
|
|
177
136
|
```
|
|
178
137
|
|
|
179
|
-
###
|
|
180
|
-
|
|
181
|
-
```javascript
|
|
182
|
-
import scan from "@1-/scan";
|
|
183
|
-
|
|
184
|
-
const dir = "./data";
|
|
185
|
-
const db_path = "./scan_record.db";
|
|
186
|
-
|
|
187
|
-
// 过滤临时文件与特定配置
|
|
188
|
-
const ignore = (kind, rel_path) => {
|
|
189
|
-
return rel_path.startsWith("temp/") || rel_path === "config.json";
|
|
190
|
-
};
|
|
191
|
-
|
|
192
|
-
const [updated_paths, upsert] = await scan(dir, db_path, ignore);
|
|
193
|
-
using _upsert = upsert;
|
|
194
|
-
|
|
195
|
-
console.log("已同步,更新列表:", updated_paths);
|
|
196
|
-
|
|
197
|
-
for (const rel_path of updated_paths) {
|
|
198
|
-
await upsert(rel_path);
|
|
199
|
-
}
|
|
200
|
-
```
|
|
201
|
-
|
|
202
|
-
### 批量存储模块使用
|
|
138
|
+
### 独立批量存储模块
|
|
203
139
|
|
|
204
140
|
```javascript
|
|
205
141
|
import save from "@1-/scan/save.js";
|
|
206
|
-
import sqlite from "@1-/
|
|
142
|
+
import sqlite from "@1-/sqlite";
|
|
207
143
|
|
|
208
144
|
const db = sqlite("./scan_record.db");
|
|
209
145
|
|
|
210
146
|
// 批量更新与删除元数据
|
|
211
|
-
save(
|
|
212
|
-
db,
|
|
213
|
-
[
|
|
214
|
-
["file.txt", new Uint8Array([1, 2, 3]), 123, 1620000000]
|
|
215
|
-
],
|
|
216
|
-
[
|
|
217
|
-
new Uint8Array([4, 5, 6])
|
|
218
|
-
]
|
|
219
|
-
);
|
|
147
|
+
save(db, [["file.txt", new Uint8Array([1, 2, 3]), 123, 1620000000]], [new Uint8Array([4, 5, 6])]);
|
|
220
148
|
|
|
221
149
|
db.close();
|
|
222
150
|
```
|
|
223
151
|
|
|
224
|
-
## 设计思路
|
|
152
|
+
## 3. 设计思路
|
|
225
153
|
|
|
226
|
-
|
|
154
|
+
系统调度各模块完成增量扫描与数据同步。
|
|
227
155
|
|
|
228
|
-
|
|
229
|
-
graph TD
|
|
230
|
-
Entry["_.js (主入口)"] -->|1. 初始化连接| Sqlite["sqlite.js"]
|
|
231
|
-
Entry -->|2. 加载已有记录| Load["load.js"]
|
|
232
|
-
Entry -->|3. 扫描文件系统并对比| DirWalk["dirWalk.js"]
|
|
233
|
-
DirWalk -->|调用| Walk["@1-/walk/walkRelIgnore"]
|
|
234
|
-
DirWalk -->|处理路径键| Hash["hash.js"]
|
|
235
|
-
Entry -->|4. 删除失效记录并返回更新函数| Trans["trans.js"]
|
|
236
|
-
Save["save.js (独立批量存储模块)"] -->|事务保障| Trans
|
|
237
|
-
```
|
|
156
|
+

|
|
238
157
|
|
|
239
|
-
1.
|
|
240
|
-
2.
|
|
241
|
-
3.
|
|
242
|
-
4.
|
|
243
|
-
5. **独立批量存储模块 (`save.js`)**:供外部调用的独立工具模块,用于在事务中批量写入与删除。
|
|
158
|
+
1. **初始化连接**:调用 `@1-/sqlite` 打开 SQLite 数据库。
|
|
159
|
+
2. **加载记录**:`load.js` 检查 `scanMtimeLen` 表是否存在,若不存在则创建。读取已记录的哈希、大小及修改时间,在内存中恢复已存在的文件映射。
|
|
160
|
+
3. **元数据比对**:遍历输入的文件列表,利用 `@1-/hash` 将路径映射为 16 字节二进制键。若大小或修改时间不一致,则加入变更列表。
|
|
161
|
+
4. **删除与返回**:使用事务批量删除已被移除的记录(已存在于数据库中但不在本次扫描列表内的文件),并返回变更路径列表与 `upsert` 函数,供外部持久化。
|
|
244
162
|
|
|
245
|
-
## 技术栈
|
|
163
|
+
## 4. 技术栈
|
|
246
164
|
|
|
247
165
|
- **Bun**:JavaScript 运行时与测试框架。
|
|
248
|
-
-
|
|
249
|
-
- **@1-/
|
|
250
|
-
- **@3-/vb**:Varint
|
|
251
|
-
- **@3-/binmap / @3-/binset
|
|
166
|
+
- **@1-/sqlite**:SQLite 连接管理与事务封装。
|
|
167
|
+
- **@1-/hash**:长度限制的 MD5 哈希工具。
|
|
168
|
+
- **@3-/vb**:Varint 变长整型编码与解码器。
|
|
169
|
+
- **@3-/binmap / @3-/binset**:基于 Rust 与 WebAssembly 的高效二进制键容器。
|
|
252
170
|
|
|
253
|
-
##
|
|
171
|
+
## 5. 代码结构
|
|
254
172
|
|
|
255
|
-
```
|
|
173
|
+
```text
|
|
256
174
|
.
|
|
257
175
|
├── src
|
|
258
|
-
│ ├── _.js #
|
|
259
|
-
│ ├──
|
|
260
|
-
│
|
|
261
|
-
|
|
262
|
-
│ ├── save.js # 独立导出的批量持久化与删除辅助函数
|
|
263
|
-
│ ├── sqlite.js # 创建并配置 SQLite 数据库实例
|
|
264
|
-
│ └── trans.js # 封装 SQLite 事务,提供异常回滚机制
|
|
265
|
-
└── tests # 单元测试目录
|
|
176
|
+
│ ├── _.js # 核心控制流程
|
|
177
|
+
│ ├── load.js # 元数据表初始化与加载
|
|
178
|
+
│ └── save.js # 批量更新与删除工具
|
|
179
|
+
└── tests # 单元测试
|
|
266
180
|
```
|
|
267
181
|
|
|
268
|
-
## 历史故事
|
|
182
|
+
## 6. 历史故事
|
|
183
|
+
|
|
184
|
+
SQLite 的诞生源自导弹驱逐舰板载损害控制软件项目。2000 年,D. Richard Hipp 为美国海军设计该系统时,遭遇商业数据库因配置复杂、无法承受断连和崩溃之痛点。Hipp 随后设计出免服务器配置、直接读写本地文件之嵌入式数据库,即 SQLite。
|
|
185
|
+
|
|
186
|
+
为节省空间与降低读写延迟,SQLite 广泛应用了 Varint(可变字节整型)编码。在这种编码下,小整数仅占用 1 字节,只有大数值才占用更多字节。本项目在内存存储设计中对文件大小与修改时间采用同样的压缩设计,契合 SQLite 节省空间与高效之设计哲学。
|
|
187
|
+
## 关于
|
|
269
188
|
|
|
270
|
-
|
|
189
|
+
本库由 [WebC.site](https://webc.site) 开发。
|
|
271
190
|
|
|
272
|
-
|
|
273
|
-
../doc/zh/about.md
|
|
191
|
+
[WebC.site](https://webc.site) : 面向人工智能的网站开发新范式
|
package/_.js
CHANGED
|
@@ -1,41 +1,59 @@
|
|
|
1
|
+
import sqlite from "@1-/sqlite";
|
|
2
|
+
import tx from "@1-/sqlite/tx.js";
|
|
1
3
|
import { BinMap } from "@3-/binmap";
|
|
2
4
|
import vbE from "@3-/vb/vbE.js";
|
|
3
|
-
import sqlite from "./sqlite.js";
|
|
4
5
|
import load from "./load.js";
|
|
5
|
-
import
|
|
6
|
-
import { stat } from "node:fs/promises";
|
|
6
|
+
import { stat as fsStat } from "node:fs/promises";
|
|
7
7
|
import { join } from "node:path";
|
|
8
8
|
import int from "@3-/int";
|
|
9
|
-
import
|
|
10
|
-
import
|
|
9
|
+
import strmd5 from "@1-/hash/strmd5.js";
|
|
10
|
+
import { BinSet } from "@3-/binset";
|
|
11
|
+
import u8eq from "@3-/u8/u8eq.js";
|
|
11
12
|
|
|
12
|
-
|
|
13
|
+
const stat = async (dir, rel_path) => {
|
|
14
|
+
const { size, mtimeMs: mtime_ms } = await fsStat(join(dir, rel_path));
|
|
15
|
+
return [size, int(mtime_ms), strmd5(rel_path)];
|
|
16
|
+
};
|
|
17
|
+
|
|
18
|
+
export default async (dir, db_path, files) => {
|
|
13
19
|
const db = sqlite(db_path),
|
|
14
20
|
existing = new BinMap(),
|
|
15
|
-
db_rows = load(db)
|
|
21
|
+
db_rows = load(db),
|
|
22
|
+
scanned = new BinSet(),
|
|
23
|
+
update = [];
|
|
16
24
|
|
|
17
25
|
db_rows.forEach(({ hash, size, mtime }) => existing.set(hash, vbE([size, mtime])));
|
|
18
26
|
|
|
19
|
-
const
|
|
20
|
-
|
|
27
|
+
for (const rel_path of files) {
|
|
28
|
+
try {
|
|
29
|
+
const [size, mtime, hash] = await stat(dir, rel_path),
|
|
30
|
+
val = existing.get(hash);
|
|
21
31
|
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
32
|
+
scanned.add(hash);
|
|
33
|
+
|
|
34
|
+
if (!val || !u8eq(val, vbE([size, mtime]))) {
|
|
35
|
+
update.push(rel_path);
|
|
36
|
+
}
|
|
37
|
+
} catch {}
|
|
27
38
|
}
|
|
28
39
|
|
|
29
|
-
const
|
|
40
|
+
const rm = db_rows.filter(({ hash }) => !scanned.has(hash)).map(({ hash }) => hash),
|
|
41
|
+
insert = db.prepare("INSERT OR REPLACE INTO scanMtimeLen(hash,size,mtime)VALUES(?,?,?)"),
|
|
30
42
|
upsert = async (rel_path) => {
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
insert.run(h, size, mtime);
|
|
43
|
+
try {
|
|
44
|
+
const [size, mtime, hash] = await stat(dir, rel_path);
|
|
45
|
+
insert.run(hash, size, mtime);
|
|
46
|
+
} catch {}
|
|
36
47
|
};
|
|
37
48
|
|
|
49
|
+
if (rm.length > 0) {
|
|
50
|
+
tx(db, () => {
|
|
51
|
+
const del = db.prepare("DELETE FROM scanMtimeLen WHERE hash=?");
|
|
52
|
+
rm.forEach((hash) => del.run(hash));
|
|
53
|
+
});
|
|
54
|
+
}
|
|
55
|
+
|
|
38
56
|
upsert[Symbol.dispose] = () => db.close();
|
|
39
57
|
|
|
40
|
-
return [
|
|
58
|
+
return [update, upsert];
|
|
41
59
|
};
|
package/package.json
CHANGED
|
@@ -1,13 +1,13 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@1-/scan",
|
|
3
|
-
"version": "0.1.
|
|
3
|
+
"version": "0.1.7",
|
|
4
4
|
"description": "Incrementally scan directory files and track metadata in SQLite / 增量扫描目录文件并使用 SQLite 记录元数据",
|
|
5
5
|
"keywords": [
|
|
6
|
-
"scan",
|
|
7
|
-
"incremental",
|
|
8
|
-
"sqlite",
|
|
9
6
|
"directory",
|
|
10
|
-
"
|
|
7
|
+
"incremental",
|
|
8
|
+
"metadata",
|
|
9
|
+
"scan",
|
|
10
|
+
"sqlite"
|
|
11
11
|
],
|
|
12
12
|
"homepage": "https://github.com/webc-site/npm/tree/main/scan",
|
|
13
13
|
"license": "MulanPSL-2.0",
|
|
@@ -21,8 +21,9 @@
|
|
|
21
21
|
".": "./_.js",
|
|
22
22
|
"./*": "./*"
|
|
23
23
|
},
|
|
24
|
-
"
|
|
25
|
-
"@1-/
|
|
24
|
+
"peerDependencies": {
|
|
25
|
+
"@1-/hash": "^0.1.0",
|
|
26
|
+
"@1-/sqlite": "^0.1.0",
|
|
26
27
|
"@3-/binmap": "^0.1.20",
|
|
27
28
|
"@3-/binset": "^0.1.6",
|
|
28
29
|
"@3-/int": "^0.1.1",
|
package/save.js
CHANGED
|
@@ -1,17 +1,17 @@
|
|
|
1
|
-
import
|
|
1
|
+
import tx from "@1-/sqlite/tx.js";
|
|
2
2
|
|
|
3
|
-
export default (db,
|
|
4
|
-
if (
|
|
5
|
-
|
|
6
|
-
if (
|
|
3
|
+
export default (db, update, rm) => {
|
|
4
|
+
if (update.length > 0 || rm.length > 0) {
|
|
5
|
+
tx(db, () => {
|
|
6
|
+
if (update.length > 0) {
|
|
7
7
|
const insert = db.prepare(
|
|
8
8
|
"INSERT OR REPLACE INTO scanMtimeLen(hash,size,mtime)VALUES(?,?,?)",
|
|
9
9
|
);
|
|
10
|
-
|
|
10
|
+
update.forEach(([_, h, size, mtime]) => insert.run(h, size, mtime));
|
|
11
11
|
}
|
|
12
|
-
if (
|
|
12
|
+
if (rm.length > 0) {
|
|
13
13
|
const del = db.prepare("DELETE FROM scanMtimeLen WHERE hash=?");
|
|
14
|
-
|
|
14
|
+
rm.forEach((h) => del.run(h));
|
|
15
15
|
}
|
|
16
16
|
});
|
|
17
17
|
}
|
package/dirWalk.js
DELETED
|
@@ -1,35 +0,0 @@
|
|
|
1
|
-
import { stat } from "node:fs/promises";
|
|
2
|
-
import { join } from "node:path";
|
|
3
|
-
import { FILE } from "@1-/walk";
|
|
4
|
-
import walkRelIgnore from "@1-/walk/walkRelIgnore.js";
|
|
5
|
-
import { BinSet } from "@3-/binset";
|
|
6
|
-
import u8eq from "@3-/u8/u8eq.js";
|
|
7
|
-
import vbE from "@3-/vb/vbE.js";
|
|
8
|
-
import int from "@3-/int";
|
|
9
|
-
import hash from "./hash.js";
|
|
10
|
-
|
|
11
|
-
export default async (dir, existing, ignore) => {
|
|
12
|
-
const scanned = new BinSet(),
|
|
13
|
-
to_update = [];
|
|
14
|
-
|
|
15
|
-
await walkRelIgnore(dir, async (kind, rel_path) => {
|
|
16
|
-
if (ignore && ignore(kind, rel_path) === false) {
|
|
17
|
-
return false;
|
|
18
|
-
}
|
|
19
|
-
if (kind === FILE) {
|
|
20
|
-
const { size, mtimeMs } = await stat(join(dir, rel_path)),
|
|
21
|
-
mtime = int(mtimeMs),
|
|
22
|
-
h = hash(rel_path);
|
|
23
|
-
|
|
24
|
-
scanned.add(h);
|
|
25
|
-
|
|
26
|
-
const val = existing.get(h);
|
|
27
|
-
if (val && u8eq(val, vbE([size, mtime]))) {
|
|
28
|
-
return;
|
|
29
|
-
}
|
|
30
|
-
to_update.push([rel_path, h, size, mtime]);
|
|
31
|
-
}
|
|
32
|
-
});
|
|
33
|
-
|
|
34
|
-
return [scanned, to_update];
|
|
35
|
-
};
|
package/hash.js
DELETED
package/sqlite.js
DELETED