EXtremeZip 2021.5.6 → 2021.5.27
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/lib/extremeunzip.zzaqsu.rb +46 -45
- data/lib/extremezip.zzaqsv.rb +191 -0
- metadata +29 -14
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA256:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: 52e17a94d2ce9cbb92c01bbd8e2ff1b14150e06e6de9842a7024475912d3c277
|
|
4
|
+
data.tar.gz: 2c72a81b8427e5934ced3f8df01f546fbd16c5b9a58a1e6ec978f9c8ce07bb48
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: e894e9a30f9fa14a915b651e9d718ca7b9e09d107843f47611d7d63c177dd42e2733ff6d36620a4685c46105bce67559c37e4d62ca1b65fd47dde8dd72922699
|
|
7
|
+
data.tar.gz: 8e121b6e3f56e5c5933a40eb90922282508006bcf144d10bc0517ed59a10154e15c162bc5ca7766627de4c31a89d0ad24532af95c1e1e7426a05ea4713ee007e
|
data/lib/extremeunzip.zzaqsu.rb
CHANGED
|
@@ -97,60 +97,61 @@ end # extractVfsDataWithVersion(wholeCbor, fileVersion) #根据版本号
|
|
|
97
97
|
class ExtremeUnZip
|
|
98
98
|
#解压
|
|
99
99
|
def exuz(rootPath)
|
|
100
|
+
wholeFileContent=File.read(rootPath) #最终文件内容
|
|
100
101
|
|
|
101
|
-
|
|
102
|
-
wholeFileContent=File.read(rootPath) #最终文件内容
|
|
103
|
-
|
|
104
|
-
# puts wholeFileContent #Debug.
|
|
105
102
|
checkMemoryUsage(60)
|
|
106
|
-
|
|
107
|
-
|
|
108
|
-
|
|
109
|
-
|
|
103
|
+
|
|
104
|
+
puts wholeFileContent.class #debug
|
|
105
|
+
|
|
106
|
+
wholeCborByteArray=wholeFileContent[4..-1] #从第5个到末尾
|
|
110
107
|
checkMemoryUsage(65)
|
|
111
|
-
|
|
112
|
-
|
|
113
|
-
# puts wholeCborByteArray #Debug.
|
|
114
|
-
|
|
115
|
-
checkMemoryUsage(70)
|
|
116
|
-
wholeCbor=CBOR.decode(wholeCborByteArray) #解码
|
|
117
|
-
# wholeCbor=wholeCborByteArray.from_cbor #解码CBOR
|
|
118
|
-
|
|
119
|
-
# puts wholeCbor #Debug.
|
|
120
|
-
checkMemoryUsage(75)
|
|
121
|
-
|
|
122
|
-
fileVersion=wholeCbor['version'] #获取版本号
|
|
123
|
-
|
|
124
|
-
puts 'fileVersion:' #Debug
|
|
125
|
-
checkMemoryUsage(80)
|
|
126
|
-
puts fileVersion #Debug.
|
|
127
|
-
|
|
128
|
-
|
|
129
|
-
if (fileVersion<14) #版本号过小
|
|
130
|
-
checkMemoryUsage(85)
|
|
131
|
-
puts 'file version too old' #报告错误
|
|
132
|
-
else #版本号够大
|
|
133
|
-
compressedVfsMenu=wholeCbor['vfsMenu'] #获取压缩后的目录内容
|
|
134
108
|
|
|
135
|
-
checkMemoryUsage(90)
|
|
136
|
-
replyByteArray=LZMA.decompress(compressedVfsMenu) #解码目录VFS字节数组内容
|
|
137
109
|
|
|
138
|
-
#
|
|
139
|
-
|
|
140
|
-
checkMemoryUsage(95)
|
|
141
|
-
# victoriaFreshData=extractVfsDataWithVersion(wholeCbor, fileVersion) #根据版本号，提取VFS数据内容
|
|
142
|
-
victoriaFreshDataFile=extractVfsDataWithVersionExternalFile(wholeCbor, fileVersion) #根据版本号，提取VFS数据内容
|
|
110
|
+
# puts wholeCborByteArray #Debug.
|
|
143
111
|
|
|
144
|
-
|
|
112
|
+
checkMemoryUsage(70)
|
|
113
|
+
wholeCbor=CBOR.decode(wholeCborByteArray) #解码
|
|
114
|
+
# wholeCbor=wholeCborByteArray.from_cbor #解码CBOR
|
|
145
115
|
|
|
146
|
-
|
|
147
|
-
|
|
116
|
+
# puts wholeCbor #Debug.
|
|
117
|
+
checkMemoryUsage(75)
|
|
148
118
|
|
|
119
|
+
fileVersion=wholeCbor['version'] #获取版本号
|
|
120
|
+
|
|
121
|
+
puts 'fileVersion:' #Debug
|
|
122
|
+
checkMemoryUsage(80)
|
|
123
|
+
puts fileVersion #Debug.
|
|
124
|
+
|
|
125
|
+
|
|
126
|
+
if (fileVersion<14) #版本号过小
|
|
127
|
+
checkMemoryUsage(85)
|
|
128
|
+
puts 'file version too old' #报告错误
|
|
129
|
+
else #版本号够大
|
|
130
|
+
compressedVfsMenu=wholeCbor['vfsMenu'] #获取压缩后的目录内容
|
|
131
|
+
|
|
132
|
+
checkMemoryUsage(90)
|
|
133
|
+
replyByteArray=LZMA.decompress(compressedVfsMenu) #解码目录VFS字节数组内容
|
|
134
|
+
|
|
135
|
+
# puts replyByteArray #Debug
|
|
136
|
+
|
|
137
|
+
checkMemoryUsage(95)
|
|
138
|
+
# victoriaFreshData=extractVfsDataWithVersion(wholeCbor, fileVersion) #根据版本号，提取VFS数据内容
|
|
139
|
+
victoriaFreshDataFile=extractVfsDataWithVersionExternalFile(wholeCbor, fileVersion) #根据版本号，提取VFS数据内容
|
|
140
|
+
|
|
141
|
+
# puts victoriaFreshData #Debug
|
|
142
|
+
|
|
143
|
+
checkMemoryUsage(100)
|
|
144
|
+
$clipDownloader=VictoriaFresh.new #创建下载器。
|
|
145
|
+
|
|
146
|
+
|
|
147
|
+
$clipDownloader.releaseFilesExternalDataFile(replyByteArray, victoriaFreshDataFile) #释放各个文件
|
|
148
|
+
|
|
149
|
+
fileToRemove=File.new(victoriaFreshDataFile) #要删除的文件
|
|
150
|
+
|
|
151
|
+
File.delete(fileToRemove) #删除文件
|
|
152
|
+
|
|
153
|
+
end #if (fileVersion<14) #版本号过小
|
|
149
154
|
|
|
150
|
-
$clipDownloader.releaseFilesExternalDataFile(replyByteArray, victoriaFreshDataFile) #释放各个文件
|
|
151
|
-
checkMemoryUsage(105)
|
|
152
|
-
end #if (fileVersion<14) #版本号过小
|
|
153
|
-
|
|
154
155
|
end #def exuz(rootPath)
|
|
155
156
|
end #class ExtremeUnZip
|
|
156
157
|
|
|
@@ -0,0 +1,191 @@
|
|
|
1
|
+
#!/usr/bin/ruby
|
|
2
|
+
|
|
3
|
+
require 'victoriafresh'
|
|
4
|
+
require 'cbor'
|
|
5
|
+
require 'lzma'
|
|
6
|
+
require 'etc' #cpu amount
|
|
7
|
+
require 'cod'
|
|
8
|
+
require 'uuid'
|
|
9
|
+
require 'get_process_mem'
|
|
10
|
+
|
|
11
|
+
def checkMemoryUsage(lineNumber)
|
|
12
|
+
mem= GetProcessMem.new
|
|
13
|
+
|
|
14
|
+
puts("#{lineNumber} , Memory: #{mem.mb}"); #Debug
|
|
15
|
+
|
|
16
|
+
end #def checkMemoryUsage
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
class ExtremeZip
|
|
20
|
+
#压缩
|
|
21
|
+
def exz(rootPath)
|
|
22
|
+
#计算一个合理的数据块长度：
|
|
23
|
+
# dataBlockLength=16777216 #数据块单元长度， 16MiB
|
|
24
|
+
dataBlockLength=33554432 #数据块单元长度， 32MiB
|
|
25
|
+
# dataBlockLength=67108864 #数据块单元长度， 64MiB
|
|
26
|
+
|
|
27
|
+
checkMemoryUsage(75)
|
|
28
|
+
|
|
29
|
+
# dynamicBlockLength=victoriaFreshData.bytesize / Etc.nprocessors + 1 #尝试根据现在的任务计算出一个动态块长度。
|
|
30
|
+
dynamicBlockLength=dataBlockLength #尝试根据现在的任务计算出一个动态块长度。
|
|
31
|
+
|
|
32
|
+
|
|
33
|
+
checkMemoryUsage(80)
|
|
34
|
+
|
|
35
|
+
maxAcceptableDataBlockLength=52852*1024 #Max acceptable data block length
|
|
36
|
+
|
|
37
|
+
dataBlockLength=[dataBlockLength, dynamicBlockLength].max #取预定义的块长度和动态块长度中较大的那个来作为块长度。这样压缩比高一些
|
|
38
|
+
checkMemoryUsage(85)
|
|
39
|
+
|
|
40
|
+
dataBlockLength=[dataBlockLength, maxAcceptableDataBlockLength].min #Limit data block length, not to exceed max acceptable data block length.
|
|
41
|
+
|
|
42
|
+
puts("block length: #{dataBlockLength}") #Debug
|
|
43
|
+
|
|
44
|
+
|
|
45
|
+
$clipDownloader=VictoriaFresh.new #创建下载器。
|
|
46
|
+
|
|
47
|
+
$clipDownloader.diskFlush=true #向磁盘写入缓存
|
|
48
|
+
$clipDownloader.diskMultiFile=true #写多个磁盘文件
|
|
49
|
+
$clipDownloader.diskFileName='victoriafreshdata.v.' #磁盘文件名前缀
|
|
50
|
+
$clipDownloader.diskFlushSize=dataBlockLength #磁盘文件大小
|
|
51
|
+
|
|
52
|
+
|
|
53
|
+
checkMemoryUsage(25)
|
|
54
|
+
victoriaFresh,victoriaFreshData=$clipDownloader.checkOnce(rootPath) #打包该目录树。
|
|
55
|
+
|
|
56
|
+
#利用protobuf打包成字节数组：
|
|
57
|
+
replyByteArray="" #回复时使用的字节数组。
|
|
58
|
+
checkMemoryUsage(30)
|
|
59
|
+
replyByteArray=victoriaFresh.to_cbor ##打包成字节数组。
|
|
60
|
+
|
|
61
|
+
checkMemoryUsage(35)
|
|
62
|
+
victoriaFreshFile=File.new("victoriafresh.v","wb") #创建文件。
|
|
63
|
+
victoriaFreshFile.syswrite(replyByteArray) #写入文件。
|
|
64
|
+
|
|
65
|
+
victoriaFreshFile.close #关闭文件。
|
|
66
|
+
checkMemoryUsage(40)
|
|
67
|
+
|
|
68
|
+
# victoriaFreshDataFile=File.new("victoriafreshdata.v","wb") #数据文件。
|
|
69
|
+
# victoriaFreshDataFile.syswrite(victoriaFreshData) #写入文件。
|
|
70
|
+
# victoriaFreshDataFile.close #关闭文件。
|
|
71
|
+
checkMemoryUsage(45)
|
|
72
|
+
|
|
73
|
+
|
|
74
|
+
wholeFileContent='' #最终文件内容
|
|
75
|
+
|
|
76
|
+
checkMemoryUsage(50)
|
|
77
|
+
wholeFileContent = wholeFileContent + 'exz' + "\0" #写入魔法文件头
|
|
78
|
+
|
|
79
|
+
wholeCbor={} #整个CBOR结构
|
|
80
|
+
wholeCbor['version']=68 #文件格式版本号
|
|
81
|
+
checkMemoryUsage(55)
|
|
82
|
+
|
|
83
|
+
uuid = UUID.new #获取生成器
|
|
84
|
+
generatedUuid=uuid.generate #生成唯一标识
|
|
85
|
+
wholeCbor['uuid']=generatedUuid #指定本个压缩包的唯一编号
|
|
86
|
+
checkMemoryUsage(60)
|
|
87
|
+
|
|
88
|
+
#压缩目录数据并放入CBOR：
|
|
89
|
+
compressedVfsMenu = LZMA.compress(replyByteArray) #压缩目录数据
|
|
90
|
+
|
|
91
|
+
checkMemoryUsage(65)
|
|
92
|
+
puts("compressed menu length: #{compressedVfsMenu.bytesize}") #Debug.
|
|
93
|
+
wholeCbor['vfsMenu']=compressedVfsMenu #加入目录
|
|
94
|
+
|
|
95
|
+
#压缩文件实体数据并放入CBOR：
|
|
96
|
+
checkMemoryUsage(70)
|
|
97
|
+
|
|
98
|
+
checkMemoryUsage(90)
|
|
99
|
+
|
|
100
|
+
|
|
101
|
+
processDataLength=0 #已处理的数据长度
|
|
102
|
+
|
|
103
|
+
checkMemoryUsage(95)
|
|
104
|
+
vfsDataList=[] #数据块压缩块列表
|
|
105
|
+
taskPipeList=[] #任务分配管道列表。
|
|
106
|
+
responsePipeList=[] #任务回复管道列表
|
|
107
|
+
processIdList = [] #记录到子进程列表中
|
|
108
|
+
checkMemoryUsage(100)
|
|
109
|
+
|
|
110
|
+
filePartAmount=$clipDownloader.currentDiskFlushSuffix #获取文件个数
|
|
111
|
+
|
|
112
|
+
filePartCounter=0 #文件计数器
|
|
113
|
+
|
|
114
|
+
while (filePartCounter < filePartAmount) #未处理完毕
|
|
115
|
+
currentBlockFile=File.new($clipDownloader.diskFileName+filePartCounter.to_s, 'rb') #打开文件
|
|
116
|
+
|
|
117
|
+
currentBlockData=currentBlockFile.read #读取全部内容
|
|
118
|
+
|
|
119
|
+
currentBlockFile.close #关闭文件
|
|
120
|
+
|
|
121
|
+
File.delete(currentBlockFile) #删除数据块文件
|
|
122
|
+
|
|
123
|
+
currentTaskPipe=Cod.pipe #任务分配管道
|
|
124
|
+
currentResponsePipe=Cod.pipe #任务回复管道
|
|
125
|
+
|
|
126
|
+
#记录管道：
|
|
127
|
+
taskPipeList << currentTaskPipe
|
|
128
|
+
responsePipeList << currentResponsePipe
|
|
129
|
+
p1 = fork do #复制出子进程
|
|
130
|
+
checkMemoryUsage(115)
|
|
131
|
+
currentBlockDataToCompress=currentBlockData #读取数据块
|
|
132
|
+
|
|
133
|
+
currentCompressedVfsData=LZMA.compress(currentBlockDataToCompress) #压缩当前块
|
|
134
|
+
|
|
135
|
+
checkMemoryUsage(120)
|
|
136
|
+
puts("compressed data length: #{currentCompressedVfsData.bytesize}") #Debug.
|
|
137
|
+
|
|
138
|
+
currentResponsePipe.put currentCompressedVfsData #将压缩后的数据块写入到回复管道中
|
|
139
|
+
|
|
140
|
+
checkMemoryUsage(125)
|
|
141
|
+
puts("finished #{Process.pid}") #Debug
|
|
142
|
+
end #p1 = fork do #复制出子进程
|
|
143
|
+
|
|
144
|
+
processDataLength=processDataLength + dataBlockLength #计数
|
|
145
|
+
checkMemoryUsage(130)
|
|
146
|
+
|
|
147
|
+
processIdList << p1 #记录到子进程列表中
|
|
148
|
+
|
|
149
|
+
filePartCounter=filePartCounter+1 #计数
|
|
150
|
+
end #while processDataLength < victoriaFreshData.byte_size do #未处理完毕
|
|
151
|
+
|
|
152
|
+
checkMemoryUsage(135)
|
|
153
|
+
processCounter=0 #子进程计数器
|
|
154
|
+
|
|
155
|
+
processIdList.each do |currentSubProcess|
|
|
156
|
+
puts("waiting #{currentSubProcess}") #Debug
|
|
157
|
+
checkMemoryUsage(140)
|
|
158
|
+
|
|
159
|
+
currentResponsePipe=responsePipeList[processCounter] #任务回复管道
|
|
160
|
+
|
|
161
|
+
currentCompressedVfsDataFromSubProcess=currentResponsePipe.get #读取压缩后数据
|
|
162
|
+
checkMemoryUsage(145)
|
|
163
|
+
|
|
164
|
+
Process.waitpid(currentSubProcess) #等待该个子进程
|
|
165
|
+
|
|
166
|
+
vfsDataList << currentCompressedVfsDataFromSubProcess #加入数据块列表中
|
|
167
|
+
checkMemoryUsage(150)
|
|
168
|
+
|
|
169
|
+
processCounter=processCounter+1 #子进程计数
|
|
170
|
+
end #processIdList.each do |currentSubProcess|
|
|
171
|
+
|
|
172
|
+
checkMemoryUsage(155)
|
|
173
|
+
wholeCbor['vfsDataList']=vfsDataList #加入数据
|
|
174
|
+
|
|
175
|
+
#序列化CBOR：
|
|
176
|
+
wholeCborByteArray=wholeCbor.to_cbor
|
|
177
|
+
checkMemoryUsage(160)
|
|
178
|
+
|
|
179
|
+
wholeFileContent = wholeFileContent + wholeCborByteArray
|
|
180
|
+
|
|
181
|
+
#写入文件：
|
|
182
|
+
checkMemoryUsage(165)
|
|
183
|
+
extremeZipOutputFile=File.new( victoriaFresh['name'] + '.exz', 'wb') #创建文件
|
|
184
|
+
extremeZipOutputFile.syswrite(wholeFileContent) #写入文件
|
|
185
|
+
extremeZipOutputFile.close #关闭文件
|
|
186
|
+
|
|
187
|
+
|
|
188
|
+
|
|
189
|
+
end #def exz(rootPath)
|
|
190
|
+
end #class ExtremeZip
|
|
191
|
+
|
metadata
CHANGED
|
@@ -1,35 +1,35 @@
|
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
|
2
2
|
name: EXtremeZip
|
|
3
3
|
version: !ruby/object:Gem::Version
|
|
4
|
-
version: 2021.5.
|
|
4
|
+
version: 2021.5.27
|
|
5
5
|
platform: ruby
|
|
6
6
|
authors:
|
|
7
7
|
- Hxcan Cai
|
|
8
|
-
autorequire:
|
|
8
|
+
autorequire:
|
|
9
9
|
bindir: bin
|
|
10
10
|
cert_chain: []
|
|
11
|
-
date: 2021-05-
|
|
11
|
+
date: 2021-05-27 00:00:00.000000000 Z
|
|
12
12
|
dependencies:
|
|
13
13
|
- !ruby/object:Gem::Dependency
|
|
14
14
|
name: VictoriaFreSh
|
|
15
15
|
requirement: !ruby/object:Gem::Requirement
|
|
16
16
|
requirements:
|
|
17
|
-
- - ">="
|
|
18
|
-
- !ruby/object:Gem::Version
|
|
19
|
-
version: 2021.5.6
|
|
20
17
|
- - "~>"
|
|
21
18
|
- !ruby/object:Gem::Version
|
|
22
|
-
version: 2021.5.
|
|
19
|
+
version: 2021.5.8
|
|
20
|
+
- - ">="
|
|
21
|
+
- !ruby/object:Gem::Version
|
|
22
|
+
version: 2021.5.8
|
|
23
23
|
type: :runtime
|
|
24
24
|
prerelease: false
|
|
25
25
|
version_requirements: !ruby/object:Gem::Requirement
|
|
26
26
|
requirements:
|
|
27
|
-
- - ">="
|
|
28
|
-
- !ruby/object:Gem::Version
|
|
29
|
-
version: 2021.5.6
|
|
30
27
|
- - "~>"
|
|
31
28
|
- !ruby/object:Gem::Version
|
|
32
|
-
version: 2021.5.
|
|
29
|
+
version: 2021.5.8
|
|
30
|
+
- - ">="
|
|
31
|
+
- !ruby/object:Gem::Version
|
|
32
|
+
version: 2021.5.8
|
|
33
33
|
- !ruby/object:Gem::Dependency
|
|
34
34
|
name: ruby-lzma
|
|
35
35
|
requirement: !ruby/object:Gem::Requirement
|
|
@@ -44,6 +44,20 @@ dependencies:
|
|
|
44
44
|
- - ">="
|
|
45
45
|
- !ruby/object:Gem::Version
|
|
46
46
|
version: 0.4.3
|
|
47
|
+
- !ruby/object:Gem::Dependency
|
|
48
|
+
name: get_process_mem
|
|
49
|
+
requirement: !ruby/object:Gem::Requirement
|
|
50
|
+
requirements:
|
|
51
|
+
- - ">="
|
|
52
|
+
- !ruby/object:Gem::Version
|
|
53
|
+
version: 0.2.7
|
|
54
|
+
type: :runtime
|
|
55
|
+
prerelease: false
|
|
56
|
+
version_requirements: !ruby/object:Gem::Requirement
|
|
57
|
+
requirements:
|
|
58
|
+
- - ">="
|
|
59
|
+
- !ruby/object:Gem::Version
|
|
60
|
+
version: 0.2.7
|
|
47
61
|
description: Extreme zip.
|
|
48
62
|
email: caihuosheng@gmail.com
|
|
49
63
|
executables:
|
|
@@ -55,11 +69,12 @@ files:
|
|
|
55
69
|
- bin/exuz
|
|
56
70
|
- bin/exz
|
|
57
71
|
- lib/extremeunzip.zzaqsu.rb
|
|
72
|
+
- lib/extremezip.zzaqsv.rb
|
|
58
73
|
homepage: http://rubygems.org/gems/EXtremeZip
|
|
59
74
|
licenses:
|
|
60
75
|
- MIT
|
|
61
76
|
metadata: {}
|
|
62
|
-
post_install_message:
|
|
77
|
+
post_install_message:
|
|
63
78
|
rdoc_options: []
|
|
64
79
|
require_paths:
|
|
65
80
|
- lib
|
|
@@ -74,8 +89,8 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
|
74
89
|
- !ruby/object:Gem::Version
|
|
75
90
|
version: '0'
|
|
76
91
|
requirements: []
|
|
77
|
-
rubygems_version: 3.
|
|
78
|
-
signing_key:
|
|
92
|
+
rubygems_version: 3.2.15
|
|
93
|
+
signing_key:
|
|
79
94
|
specification_version: 4
|
|
80
95
|
summary: EXtremeZip
|
|
81
96
|
test_files: []
|