EXtremeZip 2022.3.9 → 2023.1.29
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/bin/exz +5 -10
- data/lib/extremeunzip.zzaqsu.rb +41 -40
- data/lib/extremezip.zzaqsv.rb +140 -73
- metadata +3 -3
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 60ef4ba8a8aceb8f6be2cfb73bc763b98a293af4016429d02f49d588f2240f82
|
4
|
+
data.tar.gz: 82cc59f0b151bb1826e285a72e33bebfc3d9d0b110ffd6d41d08c96a6aedb08b
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 3b40ef64705493182c4e4dada135361fa02109bc20ac0763e598a18f9f052909a5290061b37ce46fb53d65b12cc25aa3f1e47a0fe9c5453a785d5dec0ef90865
|
7
|
+
data.tar.gz: 8f86b1fe790d74cb1e0a2b251c13a5a3324a293e16d9e9cc90571358eab54d74259cf32bf287b3d9be21789b9e907951c27fe2762135df51eb836c56b3ca967a
|
data/bin/exz
CHANGED
@@ -4,18 +4,13 @@ require 'extremezip.zzaqsv'
|
|
4
4
|
require 'notifier'
|
5
5
|
|
6
6
|
if ARGV.empty? # 未指定命令行参数。
|
7
|
+
puts("exz filename")
|
7
8
|
else # 指定了命令行参数。
|
8
|
-
|
9
|
+
$rootPath = ARGV[0] # 记录要打包的目录树的根目录。
|
9
10
|
|
10
|
-
|
11
|
+
exzObject = ExtremeZip.new # 创建压缩对象
|
11
12
|
|
12
|
-
|
13
|
-
|
14
|
-
exzObject.exz($rootPath) # 压缩
|
13
|
+
exzObject.exz($rootPath) # 压缩
|
15
14
|
|
16
|
-
|
17
|
-
image: "image.png",
|
18
|
-
title: "exz finished",
|
19
|
-
message: $rootPath
|
20
|
-
)
|
15
|
+
Notifier.notify( image: "image.png", title: "exz finished", message: $rootPath )
|
21
16
|
end
|
data/lib/extremeunzip.zzaqsu.rb
CHANGED
@@ -67,11 +67,11 @@ class ExtremeUnZip
|
|
67
67
|
checkMemoryUsage(34)
|
68
68
|
|
69
69
|
begin # 解压
|
70
|
-
|
70
|
+
currentRawData = LZMA.decompress(currentCompressed) # 解压这一块
|
71
71
|
|
72
|
-
|
72
|
+
dataFile.syswrite(currentRawData) # 写入内容
|
73
73
|
rescue RuntimeError => e # 解压失败
|
74
|
-
|
74
|
+
puts "Warning: the exz file may be incomplete." # 报告错误。文件可能不完整。
|
75
75
|
end # begin # 解压
|
76
76
|
|
77
77
|
dataBlockCounter += 1 # count
|
@@ -81,55 +81,56 @@ class ExtremeUnZip
|
|
81
81
|
end # if (fileVersion==14) #14版本
|
82
82
|
|
83
83
|
dataFileName # 返回解压后的数据块整体
|
84
|
-
end # def extractVfsDataWithVersionExternalFile(wholeCbor, fileVersion) #根据版本号,提取VFS数据内容
|
84
|
+
end # def extractVfsDataWithVersionExternalFile(wholeCbor, fileVersion) #根据版本号,提取VFS数据内容
|
85
85
|
|
86
86
|
# 解压
|
87
|
-
|
88
|
-
|
87
|
+
def exuz(rootPath)
|
88
|
+
result = true # 解压结果
|
89
89
|
|
90
|
-
|
90
|
+
currentBlockFile = File.new(rootPath, 'rb') # 打开文件
|
91
91
|
|
92
|
-
|
92
|
+
@wholeFileContent = currentBlockFile.read # 读取全部内容
|
93
93
|
|
94
|
-
|
94
|
+
currentBlockFile.close # 关闭文件
|
95
95
|
|
96
|
-
|
96
|
+
checkMemoryUsage(60)
|
97
97
|
|
98
|
-
|
98
|
+
wholeCborByteArray = @wholeFileContent[4..-1] # 从第5个到末尾
|
99
99
|
|
100
|
-
|
101
|
-
|
100
|
+
begin # 可能出错。
|
101
|
+
options = {:tolerant => true}
|
102
102
|
|
103
|
-
|
103
|
+
wholeCbor = CBOR.decode(wholeCborByteArray, options) # 解码
|
104
104
|
|
105
|
-
|
105
|
+
fileVersion = wholeCbor['version'] # 获取版本号
|
106
106
|
|
107
|
-
|
108
|
-
|
109
|
-
|
110
|
-
|
111
|
-
|
107
|
+
if (fileVersion < 14) # 版本号过小
|
108
|
+
checkMemoryUsage(85)
|
109
|
+
puts 'file version too old' # 报告错误
|
110
|
+
else # 版本号够大
|
111
|
+
compressedVfsMenu = wholeCbor['vfsMenu'] # 获取压缩后的目录内容
|
112
|
+
puts "compressed vfs menu size: #{compressedVfsMenu.size}" # D3bug
|
112
113
|
|
113
|
-
|
114
|
-
|
115
|
-
|
116
|
-
|
117
|
-
|
118
|
-
|
119
|
-
|
120
|
-
|
121
|
-
|
122
|
-
|
123
|
-
|
124
|
-
|
125
|
-
|
126
|
-
|
114
|
+
checkMemoryUsage(90)
|
115
|
+
replyByteArray = LZMA.decompress(compressedVfsMenu) # 解码目录VFS字节数组内容
|
116
|
+
|
117
|
+
checkMemoryUsage(95)
|
118
|
+
|
119
|
+
victoriaFreshDataFile = extractVfsDataWithVersionExternalFile(wholeCbor, fileVersion) # 根据版本号,提取VFS数据内容
|
120
|
+
|
121
|
+
checkMemoryUsage(100)
|
122
|
+
$clipDownloader = VictoriaFresh.new # 创建下载器。
|
123
|
+
|
124
|
+
$clipDownloader.releaseFilesExternalDataFile(replyByteArray, victoriaFreshDataFile) # 释放各个文件
|
125
|
+
|
126
|
+
fileToRemove = File.new(victoriaFreshDataFile) # 要删除的文件
|
127
|
+
end # if (fileVersion<14) #版本号过小
|
127
128
|
|
128
|
-
|
129
|
-
|
130
|
-
|
129
|
+
result =true # 解压成功
|
130
|
+
rescue EOFError => e # 文件内容提前到末尾。一般是压缩包文件未传输完全 。
|
131
|
+
puts "Error: the exz file may be incomplete." # 报告错误。文件可能不完整。
|
131
132
|
|
132
|
-
|
133
|
-
|
134
|
-
|
133
|
+
result = false # 失败
|
134
|
+
end #begin # 可能出错。
|
135
|
+
end # def exuz(rootPath)
|
135
136
|
end # class ExtremeUnZip
|
data/lib/extremezip.zzaqsv.rb
CHANGED
@@ -11,65 +11,131 @@ require 'get_process_mem'
|
|
11
11
|
require 'pathname'
|
12
12
|
|
13
13
|
def checkMemoryUsage(lineNumber)
|
14
|
-
|
14
|
+
mem = GetProcessMem.new
|
15
15
|
|
16
|
-
|
16
|
+
puts("#{lineNumber} , Memory: #{mem.mb}, process id: #{Process.pid}"); # Debug
|
17
17
|
end # def checkMemoryUsage
|
18
18
|
|
19
19
|
class ExtremeZip
|
20
|
-
|
21
|
-
|
22
|
-
|
23
|
-
|
24
|
-
|
25
|
-
|
26
|
-
|
27
|
-
|
28
|
-
|
29
|
-
|
30
|
-
|
31
|
-
|
32
|
-
|
20
|
+
def initialize
|
21
|
+
@wholeCbor = {} # 整个CBOR结构
|
22
|
+
@vfsDataList = [] # 数据块压缩块列表
|
23
|
+
@filePartCounter = 0 # 文件分块计数器
|
24
|
+
@responsePipeList = [] # 任务回复管道列表
|
25
|
+
@processIdList = [] # 子进程编号列表。
|
26
|
+
@processTimestamp = Time.new.to_i # 记录进程启动的时间戳。
|
27
|
+
@topFileList=[] # top file list. by length.
|
28
|
+
@maxSubProcessAmount = Etc.nprocessors # 获取最大的子进程个数
|
29
|
+
@leastTopFileLength=-4 # least top file length
|
30
|
+
@dataBlockLength = 33554432 # 数据块单元长度, 32MiB
|
31
|
+
|
32
|
+
@clipDownloader = VictoriaFresh.new # 创建下载器。
|
33
|
+
|
34
|
+
@clipDownloader.diskFlush = true # 向磁盘写入缓存
|
35
|
+
@clipDownloader.diskMultiFile = true # 写多个磁盘文件
|
36
|
+
@clipDownloader.diskFileName = 'victoriafreshdata.v.' # 磁盘文件名前缀
|
37
|
+
@clipDownloader.diskFlushSize = @dataBlockLength # 磁盘文件大小
|
38
|
+
@clipDownloader.ignoreFileName= '.exzignore' # 设置用于指定忽略文件列表的文件名。
|
39
|
+
end # def initialize
|
40
|
+
|
41
|
+
# report top large file list.
|
42
|
+
def reportTopLargeFileList(victoriaFresh)
|
43
|
+
#puts "vfs menu content: #{victoriaFresh}"
|
44
|
+
puts "vfs menu class: #{victoriaFresh.class}"
|
45
|
+
|
46
|
+
reportRememberOneFileNode(victoriaFresh, '.') # process one file node
|
47
|
+
|
48
|
+
puts("Top large files:")
|
49
|
+
|
50
|
+
@topFileList.each do |topFile|
|
51
|
+
puts("#{topFile['name']}, #{topFile['parent_path']}, #{topFile['file_length']}")
|
52
|
+
end
|
53
|
+
end # def reportTopLargeFileList(victoriaFresh) # report top large file list.
|
54
|
+
|
55
|
+
# process one file node
|
56
|
+
def reportRememberOneFileNode(victoriaFresh, parentPath)
|
57
|
+
if (victoriaFresh['is_file']) # it is a file
|
58
|
+
file_lenght=victoriaFresh['file_length'] # get file length
|
59
|
+
|
60
|
+
if (file_lenght > (@leastTopFileLength) ) # larger than least top file length
|
61
|
+
insertedFileObject=false
|
62
|
+
if (@topFileList.size>=10) # already have 10 files in top list
|
63
|
+
@topFileList.pop # pop last one.
|
64
|
+
end # if (@topFileList.size>=10) # already have 10 files in top list
|
65
|
+
|
66
|
+
#puts("#{__LINE__}, parent path: #{victoriaFresh['parent_path']}")
|
67
|
+
toInsertFileObject=victoriaFresh
|
68
|
+
toInsertFileObject['parent_path']=parentPath
|
69
|
+
#puts("#{__LINE__}, parent path: #{toInsertFileObject['parent_path']}, #{victoriaFresh['parent_path']}")
|
70
|
+
topFileIndex=0
|
71
|
+
#if (@topFileList.size==0) # no item in list
|
72
|
+
#puts("#{__LINE__}, insert, #{toInsertFileObject['name']}, #{toInsertFileObject['file_length']}")
|
73
|
+
#@topFileList.insert(topFileIndex, toInsertFileObject) # insert into top file list.
|
74
|
+
#else # if (@topFileList.size==0) # no item in list
|
75
|
+
#topFileIndex=@topFileList.size - 1
|
76
|
+
|
77
|
+
fileIndexRange=0..(@topFileList.size-1)
|
78
|
+
fileIndexRange.each do |topFileIndex|
|
79
|
+
#while (topFileIndex>=0) do # find insert point one by one
|
80
|
+
currentTopFile=@topFileList[topFileIndex]
|
81
|
+
|
82
|
+
if (file_lenght>currentTopFile['file_length'])
|
83
|
+
#puts("#{__LINE__}, insert, #{toInsertFileObject['name']}, #{toInsertFileObject['file_length']}, #{parentPath}, #{toInsertFileObject['parent_path']}")
|
84
|
+
@topFileList.insert(topFileIndex, toInsertFileObject) # insert into top file list.
|
85
|
+
insertedFileObject=true
|
86
|
+
break
|
87
|
+
end
|
88
|
+
|
89
|
+
#topFileIndex -= 1
|
90
|
+
end # while (topFileIndex>=0) do # find insert point one by one
|
91
|
+
#end # if (@topFileList.size==0) # no item in list
|
92
|
+
|
93
|
+
unless insertedFileObject # not inserted file object
|
94
|
+
@topFileList << (toInsertFileObject) # insert into top file list.
|
33
95
|
|
34
|
-
|
35
|
-
|
36
|
-
@
|
37
|
-
|
38
|
-
|
39
|
-
|
96
|
+
end # unless insertedFileObject # not inserted file object
|
97
|
+
|
98
|
+
@leastTopFileLength=@topFileList.last['file_length']
|
99
|
+
end # if (file_lenght>leastTopFileLength) # larger than least top file length
|
100
|
+
else # it is a directory
|
101
|
+
victoriaFresh['sub_files'].each do |sub_file|
|
102
|
+
reportRememberOneFileNode(sub_file, "#{parentPath}/#{victoriaFresh['name']}")
|
103
|
+
end
|
104
|
+
end # else # it is a directory
|
105
|
+
end # reportRememberOneFileNode(victoriaFresh) # process one file node
|
40
106
|
|
41
|
-
|
42
|
-
|
43
|
-
|
107
|
+
# 压缩目录数据。
|
108
|
+
def compressVfsMenu(victoriaFresh)
|
109
|
+
replyByteArray = victoriaFresh.to_cbor # #打包成字节数组。
|
44
110
|
|
45
|
-
|
46
|
-
|
111
|
+
# 压缩目录数据并放入CBOR:
|
112
|
+
compressedVfsMenu = LZMA.compress(replyByteArray) # 压缩目录数据
|
47
113
|
|
48
|
-
|
49
|
-
|
114
|
+
@wholeCbor['vfsMenu'] = compressedVfsMenu # 加入目录
|
115
|
+
end # compressVfsMenu(victoriaFresh) # 压缩目录数据。
|
50
116
|
|
51
|
-
|
52
|
-
|
53
|
-
|
117
|
+
# 加入基本文件信息
|
118
|
+
def addBasicFileInformation
|
119
|
+
@wholeCbor['version'] = 251 # 文件格式版本号
|
54
120
|
|
55
|
-
|
56
|
-
|
121
|
+
uuid = UUID.new # 获取生成器
|
122
|
+
@wholeCbor['uuid'] = uuid.generate # 指定本个压缩包的唯一编号
|
57
123
|
|
58
|
-
|
124
|
+
@wholeCbor['website']='https://rubygems.org/gems/EXtremeZip' # 加入网站地址
|
59
125
|
|
60
|
-
|
61
|
-
|
126
|
+
@wholeCbor['vfsDataListStart']=198910 # 压缩 VFS 数据列表起始位置。
|
127
|
+
end # addBasicFileInformation # 加入基本文件信息
|
62
128
|
|
63
|
-
|
64
|
-
|
65
|
-
|
66
|
-
|
129
|
+
def writeStamp(rootPath) # 更新时间戳文件
|
130
|
+
directoryPathName=Pathname.new(rootPath) #构造路径名字对象。
|
131
|
+
datastore= "#{directoryPathName.expand_path}/.exzstamp" # 配置文件路径
|
132
|
+
puts "writing stamp file: #{datastore}" # Debug
|
67
133
|
|
68
|
-
|
69
|
-
|
70
|
-
|
134
|
+
#stamp = wholeCbor['timestamp'] # 获取时间戳
|
135
|
+
stampCbor={} # 时间戳对象。
|
136
|
+
stampCbor['timestamp']= @processTimestamp # 设置时间戳。
|
71
137
|
|
72
|
-
|
138
|
+
compressed= stampCbor.to_cbor
|
73
139
|
|
74
140
|
extremeZipOutputFile = File.new(datastore, 'wb') # 创建文件
|
75
141
|
extremeZipOutputFile.syswrite(compressed) # 写入文件
|
@@ -154,6 +220,8 @@ class ExtremeZip
|
|
154
220
|
addBasicFileInformation # 加入基本文件信息
|
155
221
|
|
156
222
|
processIdList, responsePipeList = launchSubProcesses # 启动子进程。
|
223
|
+
|
224
|
+
reportTopLargeFileList(victoriaFresh) # report top large file list.
|
157
225
|
|
158
226
|
receiveCompressedVfsDataList(processIdList, responsePipeList) # 接收压缩后的数据块列表
|
159
227
|
|
@@ -178,7 +246,6 @@ class ExtremeZip
|
|
178
246
|
if fileExists # 存在文件,则表明将要发生增量压缩
|
179
247
|
writeStamp (rootPath) # 更新时间戳文件
|
180
248
|
end # if fileExists # 存在文件,则表明用户要求增量压缩
|
181
|
-
|
182
249
|
end # def exz(rootPath)
|
183
250
|
|
184
251
|
# 写入压缩块文件
|
@@ -206,7 +273,7 @@ class ExtremeZip
|
|
206
273
|
|
207
274
|
#wholeFileContent=File.read("comressed.#{processCounter}.cex", 'rb') # 读取压缩块。
|
208
275
|
|
209
|
-
puts "wirte file contetn length: #{wholeFileContent.length}" # Debghu
|
276
|
+
#puts "wirte file contetn length: #{wholeFileContent.length}" # Debghu
|
210
277
|
|
211
278
|
extremeZipOutputFile.syswrite(wholeFileContent) # 写入文件
|
212
279
|
|
@@ -266,58 +333,58 @@ class ExtremeZip
|
|
266
333
|
|
267
334
|
# 计划一个块的压缩计算
|
268
335
|
def schedule1Block(filePartCounter)
|
269
|
-
|
336
|
+
currentBlockData = readBlockFile(filePartCounter) # 读取块文件内容
|
270
337
|
|
271
|
-
|
338
|
+
currentResponsePipe = Cod.pipe # 任务回复管道
|
272
339
|
|
273
|
-
|
274
|
-
|
275
|
-
|
340
|
+
p1 = fork do # 复制出子进程
|
341
|
+
compressInSubProcess(currentBlockData, currentResponsePipe) # 在子进程中具体执行的压缩代码
|
342
|
+
end # p1 = fork do #复制出子进程
|
276
343
|
|
277
|
-
|
278
|
-
|
344
|
+
# processDataLength += @dataBlockLength # 计数
|
345
|
+
#checkMemoryUsage(130)
|
279
346
|
|
280
|
-
|
281
|
-
|
282
|
-
|
347
|
+
# 记录管道:
|
348
|
+
# taskPipeList << currentTaskPipe
|
349
|
+
@responsePipeList << currentResponsePipe # 记录回复管道
|
283
350
|
|
284
|
-
|
351
|
+
@processIdList << p1 # 记录到子进程列表中
|
285
352
|
|
286
|
-
|
353
|
+
@filePartCounter += 1 # 计数
|
287
354
|
|
288
|
-
|
355
|
+
[currentResponsePipe, p1]
|
289
356
|
end # schedule1Block(filePartCounter) # 计划一个块的压缩计算
|
290
357
|
|
291
358
|
# 启动子进程。
|
292
359
|
def launchSubProcesses
|
293
|
-
|
294
|
-
|
295
|
-
|
360
|
+
while ((@filePartCounter < @filePartAmount) && (@filePartCounter<@maxSubProcessAmount)) # 未处理完毕,并且未达到最大子进程个数
|
361
|
+
currentResponsePipe, p1 = schedule1Block(@filePartCounter) # 计划一个块的压缩计算
|
362
|
+
end # while processDataLength < victoriaFreshData.byte_size do #未处理完毕
|
296
363
|
|
297
|
-
|
364
|
+
[@processIdList, @responsePipeList]
|
298
365
|
end # launchSubProcesses # 启动子进程。
|
299
366
|
|
300
367
|
# 在子进程中具体执行的压缩代码
|
301
368
|
def compressInSubProcess(currentBlockData, currentResponsePipe)
|
302
|
-
|
303
|
-
|
369
|
+
#checkMemoryUsage(115) # Debug
|
370
|
+
currentBlockDataToCompress = currentBlockData # 读取数据块
|
304
371
|
|
305
|
-
|
372
|
+
currentCompressedVfsData = LZMA.compress(currentBlockDataToCompress) # 压缩当前块
|
306
373
|
|
307
|
-
|
308
|
-
|
374
|
+
#checkMemoryUsage(120)
|
375
|
+
#puts("compressed data length: #{currentCompressedVfsData.bytesize}") # Debug.
|
309
376
|
|
310
|
-
|
377
|
+
currentResponsePipe.put currentCompressedVfsData # 将压缩后的数据块写入到回复管道中
|
311
378
|
|
312
|
-
|
379
|
+
puts("finished #{Process.pid}") # Debug
|
313
380
|
end # compressInSubProcess(currentBlockData, currentResponsePipe) # 在子进程中具体执行的压缩代码
|
314
381
|
|
315
382
|
# 从子进程中读取数据,并终止子进程
|
316
383
|
def receiveFromSubProcess(currentSubProcess, responsePipeList, processCounter)
|
317
|
-
|
318
|
-
|
384
|
+
puts("waiting #{currentSubProcess}") # Debug
|
385
|
+
#checkMemoryUsage(140)
|
319
386
|
|
320
|
-
|
387
|
+
currentResponsePipe = responsePipeList[processCounter] # 任务回复管道
|
321
388
|
|
322
389
|
currentCompressedVfsDataFromSubProcess = currentResponsePipe.get # 读取压缩后数据
|
323
390
|
checkMemoryUsage(145)
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: EXtremeZip
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version:
|
4
|
+
version: 2023.1.29
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Hxcan Cai
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date:
|
11
|
+
date: 2023-01-29 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: cod
|
@@ -145,7 +145,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
145
145
|
- !ruby/object:Gem::Version
|
146
146
|
version: '0'
|
147
147
|
requirements: []
|
148
|
-
rubygems_version: 3.
|
148
|
+
rubygems_version: 3.2.33
|
149
149
|
signing_key:
|
150
150
|
specification_version: 4
|
151
151
|
summary: EXtremeZip
|