bio-velvet_underground 0.0.1
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +7 -0
- data/.document +5 -0
- data/.gitmodules +3 -0
- data/.travis.yml +13 -0
- data/Gemfile +19 -0
- data/LICENSE.txt +20 -0
- data/README.md +53 -0
- data/Rakefile +51 -0
- data/VERSION +1 -0
- data/ext/bioruby.patch +60 -0
- data/ext/mkrf_conf.rb +50 -0
- data/ext/src/Makefile +125 -0
- data/ext/src/src/allocArray.c +305 -0
- data/ext/src/src/allocArray.h +86 -0
- data/ext/src/src/autoOpen.c +107 -0
- data/ext/src/src/autoOpen.h +18 -0
- data/ext/src/src/binarySequences.c +813 -0
- data/ext/src/src/binarySequences.h +125 -0
- data/ext/src/src/concatenatedGraph.c +233 -0
- data/ext/src/src/concatenatedGraph.h +30 -0
- data/ext/src/src/concatenatedPreGraph.c +262 -0
- data/ext/src/src/concatenatedPreGraph.h +29 -0
- data/ext/src/src/correctedGraph.c +2642 -0
- data/ext/src/src/correctedGraph.h +32 -0
- data/ext/src/src/dfib.c +509 -0
- data/ext/src/src/dfib.h +69 -0
- data/ext/src/src/dfibHeap.c +89 -0
- data/ext/src/src/dfibHeap.h +39 -0
- data/ext/src/src/dfibpriv.h +105 -0
- data/ext/src/src/fib.c +628 -0
- data/ext/src/src/fib.h +78 -0
- data/ext/src/src/fibHeap.c +79 -0
- data/ext/src/src/fibHeap.h +41 -0
- data/ext/src/src/fibpriv.h +110 -0
- data/ext/src/src/globals.h +153 -0
- data/ext/src/src/graph.c +3983 -0
- data/ext/src/src/graph.h +233 -0
- data/ext/src/src/graphReConstruction.c +1472 -0
- data/ext/src/src/graphReConstruction.h +30 -0
- data/ext/src/src/graphStats.c +2167 -0
- data/ext/src/src/graphStats.h +72 -0
- data/ext/src/src/kmer.c +652 -0
- data/ext/src/src/kmer.h +73 -0
- data/ext/src/src/kmerOccurenceTable.c +236 -0
- data/ext/src/src/kmerOccurenceTable.h +44 -0
- data/ext/src/src/kseq.h +223 -0
- data/ext/src/src/locallyCorrectedGraph.c +557 -0
- data/ext/src/src/locallyCorrectedGraph.h +40 -0
- data/ext/src/src/passageMarker.c +677 -0
- data/ext/src/src/passageMarker.h +137 -0
- data/ext/src/src/preGraph.c +1717 -0
- data/ext/src/src/preGraph.h +106 -0
- data/ext/src/src/preGraphConstruction.c +990 -0
- data/ext/src/src/preGraphConstruction.h +26 -0
- data/ext/src/src/readCoherentGraph.c +557 -0
- data/ext/src/src/readCoherentGraph.h +30 -0
- data/ext/src/src/readSet.c +1734 -0
- data/ext/src/src/readSet.h +67 -0
- data/ext/src/src/recycleBin.c +199 -0
- data/ext/src/src/recycleBin.h +58 -0
- data/ext/src/src/roadMap.c +342 -0
- data/ext/src/src/roadMap.h +65 -0
- data/ext/src/src/run.c +318 -0
- data/ext/src/src/run.h +52 -0
- data/ext/src/src/run2.c +712 -0
- data/ext/src/src/scaffold.c +1876 -0
- data/ext/src/src/scaffold.h +64 -0
- data/ext/src/src/shortReadPairs.c +1243 -0
- data/ext/src/src/shortReadPairs.h +32 -0
- data/ext/src/src/splay.c +259 -0
- data/ext/src/src/splay.h +43 -0
- data/ext/src/src/splayTable.c +1315 -0
- data/ext/src/src/splayTable.h +31 -0
- data/ext/src/src/tightString.c +362 -0
- data/ext/src/src/tightString.h +82 -0
- data/ext/src/src/utility.c +199 -0
- data/ext/src/src/utility.h +98 -0
- data/ext/src/third-party/zlib-1.2.3/ChangeLog +855 -0
- data/ext/src/third-party/zlib-1.2.3/FAQ +339 -0
- data/ext/src/third-party/zlib-1.2.3/INDEX +51 -0
- data/ext/src/third-party/zlib-1.2.3/Makefile +154 -0
- data/ext/src/third-party/zlib-1.2.3/Makefile.in +154 -0
- data/ext/src/third-party/zlib-1.2.3/README +125 -0
- data/ext/src/third-party/zlib-1.2.3/adler32.c +149 -0
- data/ext/src/third-party/zlib-1.2.3/algorithm.txt +209 -0
- data/ext/src/third-party/zlib-1.2.3/amiga/Makefile.pup +66 -0
- data/ext/src/third-party/zlib-1.2.3/amiga/Makefile.sas +65 -0
- data/ext/src/third-party/zlib-1.2.3/as400/bndsrc +132 -0
- data/ext/src/third-party/zlib-1.2.3/as400/compile.clp +123 -0
- data/ext/src/third-party/zlib-1.2.3/as400/readme.txt +111 -0
- data/ext/src/third-party/zlib-1.2.3/as400/zlib.inc +331 -0
- data/ext/src/third-party/zlib-1.2.3/compress.c +79 -0
- data/ext/src/third-party/zlib-1.2.3/configure +459 -0
- data/ext/src/third-party/zlib-1.2.3/contrib/README.contrib +71 -0
- data/ext/src/third-party/zlib-1.2.3/contrib/ada/buffer_demo.adb +106 -0
- data/ext/src/third-party/zlib-1.2.3/contrib/ada/mtest.adb +156 -0
- data/ext/src/third-party/zlib-1.2.3/contrib/ada/read.adb +156 -0
- data/ext/src/third-party/zlib-1.2.3/contrib/ada/readme.txt +65 -0
- data/ext/src/third-party/zlib-1.2.3/contrib/ada/test.adb +463 -0
- data/ext/src/third-party/zlib-1.2.3/contrib/ada/zlib-streams.adb +225 -0
- data/ext/src/third-party/zlib-1.2.3/contrib/ada/zlib-streams.ads +114 -0
- data/ext/src/third-party/zlib-1.2.3/contrib/ada/zlib-thin.adb +141 -0
- data/ext/src/third-party/zlib-1.2.3/contrib/ada/zlib-thin.ads +450 -0
- data/ext/src/third-party/zlib-1.2.3/contrib/ada/zlib.adb +701 -0
- data/ext/src/third-party/zlib-1.2.3/contrib/ada/zlib.ads +328 -0
- data/ext/src/third-party/zlib-1.2.3/contrib/ada/zlib.gpr +20 -0
- data/ext/src/third-party/zlib-1.2.3/contrib/asm586/README.586 +43 -0
- data/ext/src/third-party/zlib-1.2.3/contrib/asm586/match.S +364 -0
- data/ext/src/third-party/zlib-1.2.3/contrib/asm686/README.686 +34 -0
- data/ext/src/third-party/zlib-1.2.3/contrib/asm686/match.S +329 -0
- data/ext/src/third-party/zlib-1.2.3/contrib/blast/Makefile +8 -0
- data/ext/src/third-party/zlib-1.2.3/contrib/blast/README +4 -0
- data/ext/src/third-party/zlib-1.2.3/contrib/blast/blast.c +444 -0
- data/ext/src/third-party/zlib-1.2.3/contrib/blast/blast.h +71 -0
- data/ext/src/third-party/zlib-1.2.3/contrib/blast/test.pk +0 -0
- data/ext/src/third-party/zlib-1.2.3/contrib/blast/test.txt +1 -0
- data/ext/src/third-party/zlib-1.2.3/contrib/delphi/ZLib.pas +557 -0
- data/ext/src/third-party/zlib-1.2.3/contrib/delphi/ZLibConst.pas +11 -0
- data/ext/src/third-party/zlib-1.2.3/contrib/delphi/readme.txt +76 -0
- data/ext/src/third-party/zlib-1.2.3/contrib/delphi/zlibd32.mak +93 -0
- data/ext/src/third-party/zlib-1.2.3/contrib/dotzlib/DotZLib.build +33 -0
- data/ext/src/third-party/zlib-1.2.3/contrib/dotzlib/DotZLib.chm +0 -0
- data/ext/src/third-party/zlib-1.2.3/contrib/dotzlib/DotZLib.sln +21 -0
- data/ext/src/third-party/zlib-1.2.3/contrib/dotzlib/DotZLib/AssemblyInfo.cs +58 -0
- data/ext/src/third-party/zlib-1.2.3/contrib/dotzlib/DotZLib/ChecksumImpl.cs +202 -0
- data/ext/src/third-party/zlib-1.2.3/contrib/dotzlib/DotZLib/CircularBuffer.cs +83 -0
- data/ext/src/third-party/zlib-1.2.3/contrib/dotzlib/DotZLib/CodecBase.cs +198 -0
- data/ext/src/third-party/zlib-1.2.3/contrib/dotzlib/DotZLib/Deflater.cs +106 -0
- data/ext/src/third-party/zlib-1.2.3/contrib/dotzlib/DotZLib/DotZLib.cs +288 -0
- data/ext/src/third-party/zlib-1.2.3/contrib/dotzlib/DotZLib/DotZLib.csproj +141 -0
- data/ext/src/third-party/zlib-1.2.3/contrib/dotzlib/DotZLib/GZipStream.cs +301 -0
- data/ext/src/third-party/zlib-1.2.3/contrib/dotzlib/DotZLib/Inflater.cs +105 -0
- data/ext/src/third-party/zlib-1.2.3/contrib/dotzlib/DotZLib/UnitTests.cs +274 -0
- data/ext/src/third-party/zlib-1.2.3/contrib/dotzlib/LICENSE_1_0.txt +23 -0
- data/ext/src/third-party/zlib-1.2.3/contrib/dotzlib/readme.txt +58 -0
- data/ext/src/third-party/zlib-1.2.3/contrib/infback9/README +1 -0
- data/ext/src/third-party/zlib-1.2.3/contrib/infback9/infback9.c +608 -0
- data/ext/src/third-party/zlib-1.2.3/contrib/infback9/infback9.h +37 -0
- data/ext/src/third-party/zlib-1.2.3/contrib/infback9/inffix9.h +107 -0
- data/ext/src/third-party/zlib-1.2.3/contrib/infback9/inflate9.h +47 -0
- data/ext/src/third-party/zlib-1.2.3/contrib/infback9/inftree9.c +323 -0
- data/ext/src/third-party/zlib-1.2.3/contrib/infback9/inftree9.h +55 -0
- data/ext/src/third-party/zlib-1.2.3/contrib/inflate86/inffas86.c +1157 -0
- data/ext/src/third-party/zlib-1.2.3/contrib/inflate86/inffast.S +1368 -0
- data/ext/src/third-party/zlib-1.2.3/contrib/iostream/test.cpp +24 -0
- data/ext/src/third-party/zlib-1.2.3/contrib/iostream/zfstream.cpp +329 -0
- data/ext/src/third-party/zlib-1.2.3/contrib/iostream/zfstream.h +128 -0
- data/ext/src/third-party/zlib-1.2.3/contrib/iostream2/zstream.h +307 -0
- data/ext/src/third-party/zlib-1.2.3/contrib/iostream2/zstream_test.cpp +25 -0
- data/ext/src/third-party/zlib-1.2.3/contrib/iostream3/README +35 -0
- data/ext/src/third-party/zlib-1.2.3/contrib/iostream3/TODO +17 -0
- data/ext/src/third-party/zlib-1.2.3/contrib/iostream3/test.cc +50 -0
- data/ext/src/third-party/zlib-1.2.3/contrib/iostream3/zfstream.cc +479 -0
- data/ext/src/third-party/zlib-1.2.3/contrib/iostream3/zfstream.h +466 -0
- data/ext/src/third-party/zlib-1.2.3/contrib/masm686/match.asm +413 -0
- data/ext/src/third-party/zlib-1.2.3/contrib/masmx64/bld_ml64.bat +2 -0
- data/ext/src/third-party/zlib-1.2.3/contrib/masmx64/gvmat64.asm +513 -0
- data/ext/src/third-party/zlib-1.2.3/contrib/masmx64/gvmat64.obj +0 -0
- data/ext/src/third-party/zlib-1.2.3/contrib/masmx64/inffas8664.c +186 -0
- data/ext/src/third-party/zlib-1.2.3/contrib/masmx64/inffasx64.asm +392 -0
- data/ext/src/third-party/zlib-1.2.3/contrib/masmx64/inffasx64.obj +0 -0
- data/ext/src/third-party/zlib-1.2.3/contrib/masmx64/readme.txt +28 -0
- data/ext/src/third-party/zlib-1.2.3/contrib/masmx86/bld_ml32.bat +2 -0
- data/ext/src/third-party/zlib-1.2.3/contrib/masmx86/gvmat32.asm +972 -0
- data/ext/src/third-party/zlib-1.2.3/contrib/masmx86/gvmat32.obj +0 -0
- data/ext/src/third-party/zlib-1.2.3/contrib/masmx86/gvmat32c.c +62 -0
- data/ext/src/third-party/zlib-1.2.3/contrib/masmx86/inffas32.asm +1083 -0
- data/ext/src/third-party/zlib-1.2.3/contrib/masmx86/inffas32.obj +0 -0
- data/ext/src/third-party/zlib-1.2.3/contrib/masmx86/mkasm.bat +3 -0
- data/ext/src/third-party/zlib-1.2.3/contrib/masmx86/readme.txt +21 -0
- data/ext/src/third-party/zlib-1.2.3/contrib/minizip/ChangeLogUnzip +67 -0
- data/ext/src/third-party/zlib-1.2.3/contrib/minizip/Makefile +25 -0
- data/ext/src/third-party/zlib-1.2.3/contrib/minizip/crypt.h +132 -0
- data/ext/src/third-party/zlib-1.2.3/contrib/minizip/ioapi.c +177 -0
- data/ext/src/third-party/zlib-1.2.3/contrib/minizip/ioapi.h +75 -0
- data/ext/src/third-party/zlib-1.2.3/contrib/minizip/iowin32.c +270 -0
- data/ext/src/third-party/zlib-1.2.3/contrib/minizip/iowin32.h +21 -0
- data/ext/src/third-party/zlib-1.2.3/contrib/minizip/miniunz.c +585 -0
- data/ext/src/third-party/zlib-1.2.3/contrib/minizip/minizip.c +420 -0
- data/ext/src/third-party/zlib-1.2.3/contrib/minizip/mztools.c +281 -0
- data/ext/src/third-party/zlib-1.2.3/contrib/minizip/mztools.h +31 -0
- data/ext/src/third-party/zlib-1.2.3/contrib/minizip/unzip.c +1598 -0
- data/ext/src/third-party/zlib-1.2.3/contrib/minizip/unzip.h +354 -0
- data/ext/src/third-party/zlib-1.2.3/contrib/minizip/zip.c +1219 -0
- data/ext/src/third-party/zlib-1.2.3/contrib/minizip/zip.h +235 -0
- data/ext/src/third-party/zlib-1.2.3/contrib/pascal/example.pas +599 -0
- data/ext/src/third-party/zlib-1.2.3/contrib/pascal/readme.txt +76 -0
- data/ext/src/third-party/zlib-1.2.3/contrib/pascal/zlibd32.mak +93 -0
- data/ext/src/third-party/zlib-1.2.3/contrib/pascal/zlibpas.pas +236 -0
- data/ext/src/third-party/zlib-1.2.3/contrib/puff/Makefile +8 -0
- data/ext/src/third-party/zlib-1.2.3/contrib/puff/README +63 -0
- data/ext/src/third-party/zlib-1.2.3/contrib/puff/puff.c +837 -0
- data/ext/src/third-party/zlib-1.2.3/contrib/puff/puff.h +31 -0
- data/ext/src/third-party/zlib-1.2.3/contrib/puff/zeros.raw +0 -0
- data/ext/src/third-party/zlib-1.2.3/contrib/testzlib/testzlib.c +275 -0
- data/ext/src/third-party/zlib-1.2.3/contrib/testzlib/testzlib.txt +10 -0
- data/ext/src/third-party/zlib-1.2.3/contrib/untgz/Makefile +14 -0
- data/ext/src/third-party/zlib-1.2.3/contrib/untgz/Makefile.msc +17 -0
- data/ext/src/third-party/zlib-1.2.3/contrib/untgz/untgz.c +674 -0
- data/ext/src/third-party/zlib-1.2.3/contrib/vstudio/readme.txt +73 -0
- data/ext/src/third-party/zlib-1.2.3/contrib/vstudio/vc7/miniunz.vcproj +126 -0
- data/ext/src/third-party/zlib-1.2.3/contrib/vstudio/vc7/minizip.vcproj +126 -0
- data/ext/src/third-party/zlib-1.2.3/contrib/vstudio/vc7/testzlib.vcproj +126 -0
- data/ext/src/third-party/zlib-1.2.3/contrib/vstudio/vc7/zlib.rc +32 -0
- data/ext/src/third-party/zlib-1.2.3/contrib/vstudio/vc7/zlibstat.vcproj +246 -0
- data/ext/src/third-party/zlib-1.2.3/contrib/vstudio/vc7/zlibvc.def +92 -0
- data/ext/src/third-party/zlib-1.2.3/contrib/vstudio/vc7/zlibvc.sln +78 -0
- data/ext/src/third-party/zlib-1.2.3/contrib/vstudio/vc7/zlibvc.vcproj +445 -0
- data/ext/src/third-party/zlib-1.2.3/contrib/vstudio/vc8/miniunz.vcproj +566 -0
- data/ext/src/third-party/zlib-1.2.3/contrib/vstudio/vc8/minizip.vcproj +563 -0
- data/ext/src/third-party/zlib-1.2.3/contrib/vstudio/vc8/testzlib.vcproj +948 -0
- data/ext/src/third-party/zlib-1.2.3/contrib/vstudio/vc8/testzlibdll.vcproj +567 -0
- data/ext/src/third-party/zlib-1.2.3/contrib/vstudio/vc8/zlib.rc +32 -0
- data/ext/src/third-party/zlib-1.2.3/contrib/vstudio/vc8/zlibstat.vcproj +870 -0
- data/ext/src/third-party/zlib-1.2.3/contrib/vstudio/vc8/zlibvc.def +92 -0
- data/ext/src/third-party/zlib-1.2.3/contrib/vstudio/vc8/zlibvc.sln +144 -0
- data/ext/src/third-party/zlib-1.2.3/contrib/vstudio/vc8/zlibvc.vcproj +1219 -0
- data/ext/src/third-party/zlib-1.2.3/crc32.c +423 -0
- data/ext/src/third-party/zlib-1.2.3/crc32.h +441 -0
- data/ext/src/third-party/zlib-1.2.3/deflate.c +1736 -0
- data/ext/src/third-party/zlib-1.2.3/deflate.h +331 -0
- data/ext/src/third-party/zlib-1.2.3/example.c +565 -0
- data/ext/src/third-party/zlib-1.2.3/examples/README.examples +42 -0
- data/ext/src/third-party/zlib-1.2.3/examples/fitblk.c +233 -0
- data/ext/src/third-party/zlib-1.2.3/examples/gun.c +693 -0
- data/ext/src/third-party/zlib-1.2.3/examples/gzappend.c +500 -0
- data/ext/src/third-party/zlib-1.2.3/examples/gzjoin.c +448 -0
- data/ext/src/third-party/zlib-1.2.3/examples/gzlog.c +413 -0
- data/ext/src/third-party/zlib-1.2.3/examples/gzlog.h +58 -0
- data/ext/src/third-party/zlib-1.2.3/examples/zlib_how.html +523 -0
- data/ext/src/third-party/zlib-1.2.3/examples/zpipe.c +191 -0
- data/ext/src/third-party/zlib-1.2.3/examples/zran.c +404 -0
- data/ext/src/third-party/zlib-1.2.3/gzio.c +1026 -0
- data/ext/src/third-party/zlib-1.2.3/infback.c +623 -0
- data/ext/src/third-party/zlib-1.2.3/inffast.c +318 -0
- data/ext/src/third-party/zlib-1.2.3/inffast.h +11 -0
- data/ext/src/third-party/zlib-1.2.3/inffixed.h +94 -0
- data/ext/src/third-party/zlib-1.2.3/inflate.c +1368 -0
- data/ext/src/third-party/zlib-1.2.3/inflate.h +115 -0
- data/ext/src/third-party/zlib-1.2.3/inftrees.c +329 -0
- data/ext/src/third-party/zlib-1.2.3/inftrees.h +55 -0
- data/ext/src/third-party/zlib-1.2.3/make_vms.com +461 -0
- data/ext/src/third-party/zlib-1.2.3/minigzip.c +322 -0
- data/ext/src/third-party/zlib-1.2.3/msdos/Makefile.bor +109 -0
- data/ext/src/third-party/zlib-1.2.3/msdos/Makefile.dj2 +104 -0
- data/ext/src/third-party/zlib-1.2.3/msdos/Makefile.emx +69 -0
- data/ext/src/third-party/zlib-1.2.3/msdos/Makefile.msc +106 -0
- data/ext/src/third-party/zlib-1.2.3/msdos/Makefile.tc +94 -0
- data/ext/src/third-party/zlib-1.2.3/old/Makefile.riscos +151 -0
- data/ext/src/third-party/zlib-1.2.3/old/README +3 -0
- data/ext/src/third-party/zlib-1.2.3/old/descrip.mms +48 -0
- data/ext/src/third-party/zlib-1.2.3/old/os2/Makefile.os2 +136 -0
- data/ext/src/third-party/zlib-1.2.3/old/os2/zlib.def +51 -0
- data/ext/src/third-party/zlib-1.2.3/old/visual-basic.txt +160 -0
- data/ext/src/third-party/zlib-1.2.3/old/zlib.html +971 -0
- data/ext/src/third-party/zlib-1.2.3/projects/README.projects +41 -0
- data/ext/src/third-party/zlib-1.2.3/projects/visualc6/README.txt +73 -0
- data/ext/src/third-party/zlib-1.2.3/projects/visualc6/example.dsp +278 -0
- data/ext/src/third-party/zlib-1.2.3/projects/visualc6/minigzip.dsp +278 -0
- data/ext/src/third-party/zlib-1.2.3/projects/visualc6/zlib.dsp +609 -0
- data/ext/src/third-party/zlib-1.2.3/projects/visualc6/zlib.dsw +59 -0
- data/ext/src/third-party/zlib-1.2.3/qnx/package.qpg +141 -0
- data/ext/src/third-party/zlib-1.2.3/trees.c +1219 -0
- data/ext/src/third-party/zlib-1.2.3/trees.h +128 -0
- data/ext/src/third-party/zlib-1.2.3/uncompr.c +61 -0
- data/ext/src/third-party/zlib-1.2.3/win32/DLL_FAQ.txt +397 -0
- data/ext/src/third-party/zlib-1.2.3/win32/Makefile.bor +107 -0
- data/ext/src/third-party/zlib-1.2.3/win32/Makefile.emx +69 -0
- data/ext/src/third-party/zlib-1.2.3/win32/Makefile.gcc +141 -0
- data/ext/src/third-party/zlib-1.2.3/win32/Makefile.msc +126 -0
- data/ext/src/third-party/zlib-1.2.3/win32/VisualC.txt +3 -0
- data/ext/src/third-party/zlib-1.2.3/win32/zlib.def +60 -0
- data/ext/src/third-party/zlib-1.2.3/win32/zlib1.rc +39 -0
- data/ext/src/third-party/zlib-1.2.3/zconf.h +332 -0
- data/ext/src/third-party/zlib-1.2.3/zconf.in.h +332 -0
- data/ext/src/third-party/zlib-1.2.3/zlib.3 +159 -0
- data/ext/src/third-party/zlib-1.2.3/zlib.h +1357 -0
- data/ext/src/third-party/zlib-1.2.3/zutil.c +318 -0
- data/ext/src/third-party/zlib-1.2.3/zutil.h +269 -0
- data/lib/bio-velvet_underground.rb +12 -0
- data/lib/bio-velvet_underground/external/VERSION +1 -0
- data/lib/bio-velvet_underground/velvet_underground.rb +72 -0
- data/spec/binary_sequence_store_spec.rb +27 -0
- data/spec/data/1/CnyUnifiedSeq +0 -0
- data/spec/spec_helper.rb +31 -0
- metadata +456 -0
@@ -0,0 +1,448 @@
|
|
1
|
+
/* gzjoin -- command to join gzip files into one gzip file
|
2
|
+
|
3
|
+
Copyright (C) 2004 Mark Adler, all rights reserved
|
4
|
+
version 1.0, 11 Dec 2004
|
5
|
+
|
6
|
+
This software is provided 'as-is', without any express or implied
|
7
|
+
warranty. In no event will the author be held liable for any damages
|
8
|
+
arising from the use of this software.
|
9
|
+
|
10
|
+
Permission is granted to anyone to use this software for any purpose,
|
11
|
+
including commercial applications, and to alter it and redistribute it
|
12
|
+
freely, subject to the following restrictions:
|
13
|
+
|
14
|
+
1. The origin of this software must not be misrepresented; you must not
|
15
|
+
claim that you wrote the original software. If you use this software
|
16
|
+
in a product, an acknowledgment in the product documentation would be
|
17
|
+
appreciated but is not required.
|
18
|
+
2. Altered source versions must be plainly marked as such, and must not be
|
19
|
+
misrepresented as being the original software.
|
20
|
+
3. This notice may not be removed or altered from any source distribution.
|
21
|
+
|
22
|
+
Mark Adler madler@alumni.caltech.edu
|
23
|
+
*/
|
24
|
+
|
25
|
+
/*
|
26
|
+
* Change history:
|
27
|
+
*
|
28
|
+
* 1.0 11 Dec 2004 - First version
|
29
|
+
* 1.1 12 Jun 2005 - Changed ssize_t to long for portability
|
30
|
+
*/
|
31
|
+
|
32
|
+
/*
|
33
|
+
gzjoin takes one or more gzip files on the command line and writes out a
|
34
|
+
single gzip file that will uncompress to the concatenation of the
|
35
|
+
uncompressed data from the individual gzip files. gzjoin does this without
|
36
|
+
having to recompress any of the data and without having to calculate a new
|
37
|
+
crc32 for the concatenated uncompressed data. gzjoin does however have to
|
38
|
+
decompress all of the input data in order to find the bits in the compressed
|
39
|
+
data that need to be modified to concatenate the streams.
|
40
|
+
|
41
|
+
gzjoin does not do an integrity check on the input gzip files other than
|
42
|
+
checking the gzip header and decompressing the compressed data. They are
|
43
|
+
otherwise assumed to be complete and correct.
|
44
|
+
|
45
|
+
Each joint between gzip files removes at least 18 bytes of previous trailer
|
46
|
+
and subsequent header, and inserts an average of about three bytes to the
|
47
|
+
compressed data in order to connect the streams. The output gzip file
|
48
|
+
has a minimal ten-byte gzip header with no file name or modification time.
|
49
|
+
|
50
|
+
This program was written to illustrate the use of the Z_BLOCK option of
|
51
|
+
inflate() and the crc32_combine() function. gzjoin will not compile with
|
52
|
+
versions of zlib earlier than 1.2.3.
|
53
|
+
*/
|
54
|
+
|
55
|
+
#include <stdio.h> /* fputs(), fprintf(), fwrite(), putc() */
|
56
|
+
#include <stdlib.h> /* exit(), malloc(), free() */
|
57
|
+
#include <fcntl.h> /* open() */
|
58
|
+
#include <unistd.h> /* close(), read(), lseek() */
|
59
|
+
#include "zlib.h"
|
60
|
+
/* crc32(), crc32_combine(), inflateInit2(), inflate(), inflateEnd() */
|
61
|
+
|
62
|
+
#define local static
|
63
|
+
|
64
|
+
/* exit with an error (return a value to allow use in an expression) */
|
65
|
+
local int bail(char *why1, char *why2)
|
66
|
+
{
|
67
|
+
fprintf(stderr, "gzjoin error: %s%s, output incomplete\n", why1, why2);
|
68
|
+
exit(1);
|
69
|
+
return 0;
|
70
|
+
}
|
71
|
+
|
72
|
+
/* -- simple buffered file input with access to the buffer -- */
|
73
|
+
|
74
|
+
#define CHUNK 32768 /* must be a power of two and fit in unsigned */
|
75
|
+
|
76
|
+
/* bin buffered input file type */
|
77
|
+
typedef struct {
|
78
|
+
char *name; /* name of file for error messages */
|
79
|
+
int fd; /* file descriptor */
|
80
|
+
unsigned left; /* bytes remaining at next */
|
81
|
+
unsigned char *next; /* next byte to read */
|
82
|
+
unsigned char *buf; /* allocated buffer of length CHUNK */
|
83
|
+
} bin;
|
84
|
+
|
85
|
+
/* close a buffered file and free allocated memory */
|
86
|
+
local void bclose(bin *in)
|
87
|
+
{
|
88
|
+
if (in != NULL) {
|
89
|
+
if (in->fd != -1)
|
90
|
+
close(in->fd);
|
91
|
+
if (in->buf != NULL)
|
92
|
+
free(in->buf);
|
93
|
+
free(in);
|
94
|
+
}
|
95
|
+
}
|
96
|
+
|
97
|
+
/* open a buffered file for input, return a pointer to type bin, or NULL on
|
98
|
+
failure */
|
99
|
+
local bin *bopen(char *name)
|
100
|
+
{
|
101
|
+
bin *in;
|
102
|
+
|
103
|
+
in = malloc(sizeof(bin));
|
104
|
+
if (in == NULL)
|
105
|
+
return NULL;
|
106
|
+
in->buf = malloc(CHUNK);
|
107
|
+
in->fd = open(name, O_RDONLY, 0);
|
108
|
+
if (in->buf == NULL || in->fd == -1) {
|
109
|
+
bclose(in);
|
110
|
+
return NULL;
|
111
|
+
}
|
112
|
+
in->left = 0;
|
113
|
+
in->next = in->buf;
|
114
|
+
in->name = name;
|
115
|
+
return in;
|
116
|
+
}
|
117
|
+
|
118
|
+
/* load buffer from file, return -1 on read error, 0 or 1 on success, with
|
119
|
+
1 indicating that end-of-file was reached */
|
120
|
+
local int bload(bin *in)
|
121
|
+
{
|
122
|
+
long len;
|
123
|
+
|
124
|
+
if (in == NULL)
|
125
|
+
return -1;
|
126
|
+
if (in->left != 0)
|
127
|
+
return 0;
|
128
|
+
in->next = in->buf;
|
129
|
+
do {
|
130
|
+
len = (long)read(in->fd, in->buf + in->left, CHUNK - in->left);
|
131
|
+
if (len < 0)
|
132
|
+
return -1;
|
133
|
+
in->left += (unsigned)len;
|
134
|
+
} while (len != 0 && in->left < CHUNK);
|
135
|
+
return len == 0 ? 1 : 0;
|
136
|
+
}
|
137
|
+
|
138
|
+
/* get a byte from the file, bail if end of file */
|
139
|
+
#define bget(in) (in->left ? 0 : bload(in), \
|
140
|
+
in->left ? (in->left--, *(in->next)++) : \
|
141
|
+
bail("unexpected end of file on ", in->name))
|
142
|
+
|
143
|
+
/* get a four-byte little-endian unsigned integer from file */
|
144
|
+
local unsigned long bget4(bin *in)
|
145
|
+
{
|
146
|
+
unsigned long val;
|
147
|
+
|
148
|
+
val = bget(in);
|
149
|
+
val += (unsigned long)(bget(in)) << 8;
|
150
|
+
val += (unsigned long)(bget(in)) << 16;
|
151
|
+
val += (unsigned long)(bget(in)) << 24;
|
152
|
+
return val;
|
153
|
+
}
|
154
|
+
|
155
|
+
/* skip bytes in file */
|
156
|
+
local void bskip(bin *in, unsigned skip)
|
157
|
+
{
|
158
|
+
/* check pointer */
|
159
|
+
if (in == NULL)
|
160
|
+
return;
|
161
|
+
|
162
|
+
/* easy case -- skip bytes in buffer */
|
163
|
+
if (skip <= in->left) {
|
164
|
+
in->left -= skip;
|
165
|
+
in->next += skip;
|
166
|
+
return;
|
167
|
+
}
|
168
|
+
|
169
|
+
/* skip what's in buffer, discard buffer contents */
|
170
|
+
skip -= in->left;
|
171
|
+
in->left = 0;
|
172
|
+
|
173
|
+
/* seek past multiples of CHUNK bytes */
|
174
|
+
if (skip > CHUNK) {
|
175
|
+
unsigned left;
|
176
|
+
|
177
|
+
left = skip & (CHUNK - 1);
|
178
|
+
if (left == 0) {
|
179
|
+
/* exact number of chunks: seek all the way minus one byte to check
|
180
|
+
for end-of-file with a read */
|
181
|
+
lseek(in->fd, skip - 1, SEEK_CUR);
|
182
|
+
if (read(in->fd, in->buf, 1) != 1)
|
183
|
+
bail("unexpected end of file on ", in->name);
|
184
|
+
return;
|
185
|
+
}
|
186
|
+
|
187
|
+
/* skip the integral chunks, update skip with remainder */
|
188
|
+
lseek(in->fd, skip - left, SEEK_CUR);
|
189
|
+
skip = left;
|
190
|
+
}
|
191
|
+
|
192
|
+
/* read more input and skip remainder */
|
193
|
+
bload(in);
|
194
|
+
if (skip > in->left)
|
195
|
+
bail("unexpected end of file on ", in->name);
|
196
|
+
in->left -= skip;
|
197
|
+
in->next += skip;
|
198
|
+
}
|
199
|
+
|
200
|
+
/* -- end of buffered input functions -- */
|
201
|
+
|
202
|
+
/* skip the gzip header from file in */
|
203
|
+
local void gzhead(bin *in)
|
204
|
+
{
|
205
|
+
int flags;
|
206
|
+
|
207
|
+
/* verify gzip magic header and compression method */
|
208
|
+
if (bget(in) != 0x1f || bget(in) != 0x8b || bget(in) != 8)
|
209
|
+
bail(in->name, " is not a valid gzip file");
|
210
|
+
|
211
|
+
/* get and verify flags */
|
212
|
+
flags = bget(in);
|
213
|
+
if ((flags & 0xe0) != 0)
|
214
|
+
bail("unknown reserved bits set in ", in->name);
|
215
|
+
|
216
|
+
/* skip modification time, extra flags, and os */
|
217
|
+
bskip(in, 6);
|
218
|
+
|
219
|
+
/* skip extra field if present */
|
220
|
+
if (flags & 4) {
|
221
|
+
unsigned len;
|
222
|
+
|
223
|
+
len = bget(in);
|
224
|
+
len += (unsigned)(bget(in)) << 8;
|
225
|
+
bskip(in, len);
|
226
|
+
}
|
227
|
+
|
228
|
+
/* skip file name if present */
|
229
|
+
if (flags & 8)
|
230
|
+
while (bget(in) != 0)
|
231
|
+
;
|
232
|
+
|
233
|
+
/* skip comment if present */
|
234
|
+
if (flags & 16)
|
235
|
+
while (bget(in) != 0)
|
236
|
+
;
|
237
|
+
|
238
|
+
/* skip header crc if present */
|
239
|
+
if (flags & 2)
|
240
|
+
bskip(in, 2);
|
241
|
+
}
|
242
|
+
|
243
|
+
/* write a four-byte little-endian unsigned integer to out */
|
244
|
+
local void put4(unsigned long val, FILE *out)
|
245
|
+
{
|
246
|
+
putc(val & 0xff, out);
|
247
|
+
putc((val >> 8) & 0xff, out);
|
248
|
+
putc((val >> 16) & 0xff, out);
|
249
|
+
putc((val >> 24) & 0xff, out);
|
250
|
+
}
|
251
|
+
|
252
|
+
/* Load up zlib stream from buffered input, bail if end of file */
|
253
|
+
local void zpull(z_streamp strm, bin *in)
|
254
|
+
{
|
255
|
+
if (in->left == 0)
|
256
|
+
bload(in);
|
257
|
+
if (in->left == 0)
|
258
|
+
bail("unexpected end of file on ", in->name);
|
259
|
+
strm->avail_in = in->left;
|
260
|
+
strm->next_in = in->next;
|
261
|
+
}
|
262
|
+
|
263
|
+
/* Write header for gzip file to out and initialize trailer. */
|
264
|
+
local void gzinit(unsigned long *crc, unsigned long *tot, FILE *out)
|
265
|
+
{
|
266
|
+
fwrite("\x1f\x8b\x08\0\0\0\0\0\0\xff", 1, 10, out);
|
267
|
+
*crc = crc32(0L, Z_NULL, 0);
|
268
|
+
*tot = 0;
|
269
|
+
}
|
270
|
+
|
271
|
+
/* Copy the compressed data from name, zeroing the last block bit of the last
|
272
|
+
block if clr is true, and adding empty blocks as needed to get to a byte
|
273
|
+
boundary. If clr is false, then the last block becomes the last block of
|
274
|
+
the output, and the gzip trailer is written. crc and tot maintains the
|
275
|
+
crc and length (modulo 2^32) of the output for the trailer. The resulting
|
276
|
+
gzip file is written to out. gzinit() must be called before the first call
|
277
|
+
of gzcopy() to write the gzip header and to initialize crc and tot. */
|
278
|
+
local void gzcopy(char *name, int clr, unsigned long *crc, unsigned long *tot,
|
279
|
+
FILE *out)
|
280
|
+
{
|
281
|
+
int ret; /* return value from zlib functions */
|
282
|
+
int pos; /* where the "last block" bit is in byte */
|
283
|
+
int last; /* true if processing the last block */
|
284
|
+
bin *in; /* buffered input file */
|
285
|
+
unsigned char *start; /* start of compressed data in buffer */
|
286
|
+
unsigned char *junk; /* buffer for uncompressed data -- discarded */
|
287
|
+
z_off_t len; /* length of uncompressed data (support > 4 GB) */
|
288
|
+
z_stream strm; /* zlib inflate stream */
|
289
|
+
|
290
|
+
/* open gzip file and skip header */
|
291
|
+
in = bopen(name);
|
292
|
+
if (in == NULL)
|
293
|
+
bail("could not open ", name);
|
294
|
+
gzhead(in);
|
295
|
+
|
296
|
+
/* allocate buffer for uncompressed data and initialize raw inflate
|
297
|
+
stream */
|
298
|
+
junk = malloc(CHUNK);
|
299
|
+
strm.zalloc = Z_NULL;
|
300
|
+
strm.zfree = Z_NULL;
|
301
|
+
strm.opaque = Z_NULL;
|
302
|
+
strm.avail_in = 0;
|
303
|
+
strm.next_in = Z_NULL;
|
304
|
+
ret = inflateInit2(&strm, -15);
|
305
|
+
if (junk == NULL || ret != Z_OK)
|
306
|
+
bail("out of memory", "");
|
307
|
+
|
308
|
+
/* inflate and copy compressed data, clear last-block bit if requested */
|
309
|
+
len = 0;
|
310
|
+
zpull(&strm, in);
|
311
|
+
start = strm.next_in;
|
312
|
+
last = start[0] & 1;
|
313
|
+
if (last && clr)
|
314
|
+
start[0] &= ~1;
|
315
|
+
strm.avail_out = 0;
|
316
|
+
for (;;) {
|
317
|
+
/* if input used and output done, write used input and get more */
|
318
|
+
if (strm.avail_in == 0 && strm.avail_out != 0) {
|
319
|
+
fwrite(start, 1, strm.next_in - start, out);
|
320
|
+
start = in->buf;
|
321
|
+
in->left = 0;
|
322
|
+
zpull(&strm, in);
|
323
|
+
}
|
324
|
+
|
325
|
+
/* decompress -- return early when end-of-block reached */
|
326
|
+
strm.avail_out = CHUNK;
|
327
|
+
strm.next_out = junk;
|
328
|
+
ret = inflate(&strm, Z_BLOCK);
|
329
|
+
switch (ret) {
|
330
|
+
case Z_MEM_ERROR:
|
331
|
+
bail("out of memory", "");
|
332
|
+
case Z_DATA_ERROR:
|
333
|
+
bail("invalid compressed data in ", in->name);
|
334
|
+
}
|
335
|
+
|
336
|
+
/* update length of uncompressed data */
|
337
|
+
len += CHUNK - strm.avail_out;
|
338
|
+
|
339
|
+
/* check for block boundary (only get this when block copied out) */
|
340
|
+
if (strm.data_type & 128) {
|
341
|
+
/* if that was the last block, then done */
|
342
|
+
if (last)
|
343
|
+
break;
|
344
|
+
|
345
|
+
/* number of unused bits in last byte */
|
346
|
+
pos = strm.data_type & 7;
|
347
|
+
|
348
|
+
/* find the next last-block bit */
|
349
|
+
if (pos != 0) {
|
350
|
+
/* next last-block bit is in last used byte */
|
351
|
+
pos = 0x100 >> pos;
|
352
|
+
last = strm.next_in[-1] & pos;
|
353
|
+
if (last && clr)
|
354
|
+
strm.next_in[-1] &= ~pos;
|
355
|
+
}
|
356
|
+
else {
|
357
|
+
/* next last-block bit is in next unused byte */
|
358
|
+
if (strm.avail_in == 0) {
|
359
|
+
/* don't have that byte yet -- get it */
|
360
|
+
fwrite(start, 1, strm.next_in - start, out);
|
361
|
+
start = in->buf;
|
362
|
+
in->left = 0;
|
363
|
+
zpull(&strm, in);
|
364
|
+
}
|
365
|
+
last = strm.next_in[0] & 1;
|
366
|
+
if (last && clr)
|
367
|
+
strm.next_in[0] &= ~1;
|
368
|
+
}
|
369
|
+
}
|
370
|
+
}
|
371
|
+
|
372
|
+
/* update buffer with unused input */
|
373
|
+
in->left = strm.avail_in;
|
374
|
+
in->next = strm.next_in;
|
375
|
+
|
376
|
+
/* copy used input, write empty blocks to get to byte boundary */
|
377
|
+
pos = strm.data_type & 7;
|
378
|
+
fwrite(start, 1, in->next - start - 1, out);
|
379
|
+
last = in->next[-1];
|
380
|
+
if (pos == 0 || !clr)
|
381
|
+
/* already at byte boundary, or last file: write last byte */
|
382
|
+
putc(last, out);
|
383
|
+
else {
|
384
|
+
/* append empty blocks to last byte */
|
385
|
+
last &= ((0x100 >> pos) - 1); /* assure unused bits are zero */
|
386
|
+
if (pos & 1) {
|
387
|
+
/* odd -- append an empty stored block */
|
388
|
+
putc(last, out);
|
389
|
+
if (pos == 1)
|
390
|
+
putc(0, out); /* two more bits in block header */
|
391
|
+
fwrite("\0\0\xff\xff", 1, 4, out);
|
392
|
+
}
|
393
|
+
else {
|
394
|
+
/* even -- append 1, 2, or 3 empty fixed blocks */
|
395
|
+
switch (pos) {
|
396
|
+
case 6:
|
397
|
+
putc(last | 8, out);
|
398
|
+
last = 0;
|
399
|
+
case 4:
|
400
|
+
putc(last | 0x20, out);
|
401
|
+
last = 0;
|
402
|
+
case 2:
|
403
|
+
putc(last | 0x80, out);
|
404
|
+
putc(0, out);
|
405
|
+
}
|
406
|
+
}
|
407
|
+
}
|
408
|
+
|
409
|
+
/* update crc and tot */
|
410
|
+
*crc = crc32_combine(*crc, bget4(in), len);
|
411
|
+
*tot += (unsigned long)len;
|
412
|
+
|
413
|
+
/* clean up */
|
414
|
+
inflateEnd(&strm);
|
415
|
+
free(junk);
|
416
|
+
bclose(in);
|
417
|
+
|
418
|
+
/* write trailer if this is the last gzip file */
|
419
|
+
if (!clr) {
|
420
|
+
put4(*crc, out);
|
421
|
+
put4(*tot, out);
|
422
|
+
}
|
423
|
+
}
|
424
|
+
|
425
|
+
/* join the gzip files on the command line, write result to stdout */
|
426
|
+
int main(int argc, char **argv)
|
427
|
+
{
|
428
|
+
unsigned long crc, tot; /* running crc and total uncompressed length */
|
429
|
+
|
430
|
+
/* skip command name */
|
431
|
+
argc--;
|
432
|
+
argv++;
|
433
|
+
|
434
|
+
/* show usage if no arguments */
|
435
|
+
if (argc == 0) {
|
436
|
+
fputs("gzjoin usage: gzjoin f1.gz [f2.gz [f3.gz ...]] > fjoin.gz\n",
|
437
|
+
stderr);
|
438
|
+
return 0;
|
439
|
+
}
|
440
|
+
|
441
|
+
/* join gzip files on command line and write to stdout */
|
442
|
+
gzinit(&crc, &tot, stdout);
|
443
|
+
while (argc--)
|
444
|
+
gzcopy(*argv++, argc, &crc, &tot, stdout);
|
445
|
+
|
446
|
+
/* done */
|
447
|
+
return 0;
|
448
|
+
}
|
@@ -0,0 +1,413 @@
|
|
1
|
+
/*
|
2
|
+
* gzlog.c
|
3
|
+
* Copyright (C) 2004 Mark Adler
|
4
|
+
* For conditions of distribution and use, see copyright notice in gzlog.h
|
5
|
+
* version 1.0, 26 Nov 2004
|
6
|
+
*
|
7
|
+
*/
|
8
|
+
|
9
|
+
#include <string.h> /* memcmp() */
|
10
|
+
#include <stdlib.h> /* malloc(), free(), NULL */
|
11
|
+
#include <sys/types.h> /* size_t, off_t */
|
12
|
+
#include <unistd.h> /* read(), close(), sleep(), ftruncate(), */
|
13
|
+
/* lseek() */
|
14
|
+
#include <fcntl.h> /* open() */
|
15
|
+
#include <sys/file.h> /* flock() */
|
16
|
+
#include "zlib.h" /* deflateInit2(), deflate(), deflateEnd() */
|
17
|
+
|
18
|
+
#include "gzlog.h" /* interface */
|
19
|
+
#define local static
|
20
|
+
|
21
|
+
/* log object structure */
|
22
|
+
typedef struct {
|
23
|
+
int id; /* object identifier */
|
24
|
+
int fd; /* log file descriptor */
|
25
|
+
off_t extra; /* offset of extra "ap" subfield */
|
26
|
+
off_t mark_off; /* offset of marked data */
|
27
|
+
off_t last_off; /* offset of last block */
|
28
|
+
unsigned long crc; /* uncompressed crc */
|
29
|
+
unsigned long len; /* uncompressed length (modulo 2^32) */
|
30
|
+
unsigned stored; /* length of current stored block */
|
31
|
+
} gz_log;
|
32
|
+
|
33
|
+
#define GZLOGID 19334 /* gz_log object identifier */
|
34
|
+
|
35
|
+
#define LOCK_RETRY 1 /* retry lock once a second */
|
36
|
+
#define LOCK_PATIENCE 1200 /* try about twenty minutes before forcing */
|
37
|
+
|
38
|
+
/* acquire a lock on a file */
|
39
|
+
local int lock(int fd)
|
40
|
+
{
|
41
|
+
int patience;
|
42
|
+
|
43
|
+
/* try to lock every LOCK_RETRY seconds for LOCK_PATIENCE seconds */
|
44
|
+
patience = LOCK_PATIENCE;
|
45
|
+
do {
|
46
|
+
if (flock(fd, LOCK_EX + LOCK_NB) == 0)
|
47
|
+
return 0;
|
48
|
+
(void)sleep(LOCK_RETRY);
|
49
|
+
patience -= LOCK_RETRY;
|
50
|
+
} while (patience > 0);
|
51
|
+
|
52
|
+
/* we've run out of patience -- give up */
|
53
|
+
return -1;
|
54
|
+
}
|
55
|
+
|
56
|
+
/* release lock */
|
57
|
+
local void unlock(int fd)
|
58
|
+
{
|
59
|
+
(void)flock(fd, LOCK_UN);
|
60
|
+
}
|
61
|
+
|
62
|
+
/* release a log object */
|
63
|
+
local void log_clean(gz_log *log)
|
64
|
+
{
|
65
|
+
unlock(log->fd);
|
66
|
+
(void)close(log->fd);
|
67
|
+
free(log);
|
68
|
+
}
|
69
|
+
|
70
|
+
/* read an unsigned long from a byte buffer little-endian */
|
71
|
+
local unsigned long make_ulg(unsigned char *buf)
|
72
|
+
{
|
73
|
+
int n;
|
74
|
+
unsigned long val;
|
75
|
+
|
76
|
+
val = (unsigned long)(*buf++);
|
77
|
+
for (n = 8; n < 32; n += 8)
|
78
|
+
val += (unsigned long)(*buf++) << n;
|
79
|
+
return val;
|
80
|
+
}
|
81
|
+
|
82
|
+
/* read an off_t from a byte buffer little-endian */
|
83
|
+
local off_t make_off(unsigned char *buf)
|
84
|
+
{
|
85
|
+
int n;
|
86
|
+
off_t val;
|
87
|
+
|
88
|
+
val = (off_t)(*buf++);
|
89
|
+
for (n = 8; n < 64; n += 8)
|
90
|
+
val += (off_t)(*buf++) << n;
|
91
|
+
return val;
|
92
|
+
}
|
93
|
+
|
94
|
+
/* write an unsigned long little-endian to byte buffer */
|
95
|
+
local void dice_ulg(unsigned long val, unsigned char *buf)
|
96
|
+
{
|
97
|
+
int n;
|
98
|
+
|
99
|
+
for (n = 0; n < 4; n++) {
|
100
|
+
*buf++ = val & 0xff;
|
101
|
+
val >>= 8;
|
102
|
+
}
|
103
|
+
}
|
104
|
+
|
105
|
+
/* write an off_t little-endian to byte buffer */
|
106
|
+
local void dice_off(off_t val, unsigned char *buf)
|
107
|
+
{
|
108
|
+
int n;
|
109
|
+
|
110
|
+
for (n = 0; n < 8; n++) {
|
111
|
+
*buf++ = val & 0xff;
|
112
|
+
val >>= 8;
|
113
|
+
}
|
114
|
+
}
|
115
|
+
|
116
|
+
/* initial, empty gzip file for appending */
|
117
|
+
local char empty_gz[] = {
|
118
|
+
0x1f, 0x8b, /* magic gzip id */
|
119
|
+
8, /* compression method is deflate */
|
120
|
+
4, /* there is an extra field */
|
121
|
+
0, 0, 0, 0, /* no modification time provided */
|
122
|
+
0, 0xff, /* no extra flags, no OS */
|
123
|
+
20, 0, 'a', 'p', 16, 0, /* extra field with "ap" subfield */
|
124
|
+
32, 0, 0, 0, 0, 0, 0, 0, /* offset of uncompressed data */
|
125
|
+
32, 0, 0, 0, 0, 0, 0, 0, /* offset of last block */
|
126
|
+
1, 0, 0, 0xff, 0xff, /* empty stored block (last) */
|
127
|
+
0, 0, 0, 0, /* crc */
|
128
|
+
0, 0, 0, 0 /* uncompressed length */
|
129
|
+
};
|
130
|
+
|
131
|
+
/* initialize a log object with locking */
|
132
|
+
void *gzlog_open(char *path)
|
133
|
+
{
|
134
|
+
unsigned xlen;
|
135
|
+
unsigned char temp[20];
|
136
|
+
unsigned sub_len;
|
137
|
+
int good;
|
138
|
+
gz_log *log;
|
139
|
+
|
140
|
+
/* allocate log structure */
|
141
|
+
log = malloc(sizeof(gz_log));
|
142
|
+
if (log == NULL)
|
143
|
+
return NULL;
|
144
|
+
log->id = GZLOGID;
|
145
|
+
|
146
|
+
/* open file, creating it if necessary, and locking it */
|
147
|
+
log->fd = open(path, O_RDWR | O_CREAT, 0600);
|
148
|
+
if (log->fd < 0) {
|
149
|
+
free(log);
|
150
|
+
return NULL;
|
151
|
+
}
|
152
|
+
if (lock(log->fd)) {
|
153
|
+
close(log->fd);
|
154
|
+
free(log);
|
155
|
+
return NULL;
|
156
|
+
}
|
157
|
+
|
158
|
+
/* if file is empty, write new gzip stream */
|
159
|
+
if (lseek(log->fd, 0, SEEK_END) == 0) {
|
160
|
+
if (write(log->fd, empty_gz, sizeof(empty_gz)) != sizeof(empty_gz)) {
|
161
|
+
log_clean(log);
|
162
|
+
return NULL;
|
163
|
+
}
|
164
|
+
}
|
165
|
+
|
166
|
+
/* check gzip header */
|
167
|
+
(void)lseek(log->fd, 0, SEEK_SET);
|
168
|
+
if (read(log->fd, temp, 12) != 12 || temp[0] != 0x1f ||
|
169
|
+
temp[1] != 0x8b || temp[2] != 8 || (temp[3] & 4) == 0) {
|
170
|
+
log_clean(log);
|
171
|
+
return NULL;
|
172
|
+
}
|
173
|
+
|
174
|
+
/* process extra field to find "ap" sub-field */
|
175
|
+
xlen = temp[10] + (temp[11] << 8);
|
176
|
+
good = 0;
|
177
|
+
while (xlen) {
|
178
|
+
if (xlen < 4 || read(log->fd, temp, 4) != 4)
|
179
|
+
break;
|
180
|
+
sub_len = temp[2];
|
181
|
+
sub_len += temp[3] << 8;
|
182
|
+
xlen -= 4;
|
183
|
+
if (memcmp(temp, "ap", 2) == 0 && sub_len == 16) {
|
184
|
+
good = 1;
|
185
|
+
break;
|
186
|
+
}
|
187
|
+
if (xlen < sub_len)
|
188
|
+
break;
|
189
|
+
(void)lseek(log->fd, sub_len, SEEK_CUR);
|
190
|
+
xlen -= sub_len;
|
191
|
+
}
|
192
|
+
if (!good) {
|
193
|
+
log_clean(log);
|
194
|
+
return NULL;
|
195
|
+
}
|
196
|
+
|
197
|
+
/* read in "ap" sub-field */
|
198
|
+
log->extra = lseek(log->fd, 0, SEEK_CUR);
|
199
|
+
if (read(log->fd, temp, 16) != 16) {
|
200
|
+
log_clean(log);
|
201
|
+
return NULL;
|
202
|
+
}
|
203
|
+
log->mark_off = make_off(temp);
|
204
|
+
log->last_off = make_off(temp + 8);
|
205
|
+
|
206
|
+
/* get crc, length of gzip file */
|
207
|
+
(void)lseek(log->fd, log->last_off, SEEK_SET);
|
208
|
+
if (read(log->fd, temp, 13) != 13 ||
|
209
|
+
memcmp(temp, "\001\000\000\377\377", 5) != 0) {
|
210
|
+
log_clean(log);
|
211
|
+
return NULL;
|
212
|
+
}
|
213
|
+
log->crc = make_ulg(temp + 5);
|
214
|
+
log->len = make_ulg(temp + 9);
|
215
|
+
|
216
|
+
/* set up to write over empty last block */
|
217
|
+
(void)lseek(log->fd, log->last_off + 5, SEEK_SET);
|
218
|
+
log->stored = 0;
|
219
|
+
return (void *)log;
|
220
|
+
}
|
221
|
+
|
222
|
+
/* maximum amount to put in a stored block before starting a new one */
|
223
|
+
#define MAX_BLOCK 16384
|
224
|
+
|
225
|
+
/* write a block to a log object */
|
226
|
+
int gzlog_write(void *obj, char *data, size_t len)
|
227
|
+
{
|
228
|
+
size_t some;
|
229
|
+
unsigned char temp[5];
|
230
|
+
gz_log *log;
|
231
|
+
|
232
|
+
/* check object */
|
233
|
+
log = (gz_log *)obj;
|
234
|
+
if (log == NULL || log->id != GZLOGID)
|
235
|
+
return 1;
|
236
|
+
|
237
|
+
/* write stored blocks until all of the input is written */
|
238
|
+
do {
|
239
|
+
some = MAX_BLOCK - log->stored;
|
240
|
+
if (some > len)
|
241
|
+
some = len;
|
242
|
+
if (write(log->fd, data, some) != some)
|
243
|
+
return 1;
|
244
|
+
log->crc = crc32(log->crc, data, some);
|
245
|
+
log->len += some;
|
246
|
+
len -= some;
|
247
|
+
data += some;
|
248
|
+
log->stored += some;
|
249
|
+
|
250
|
+
/* if the stored block is full, end it and start another */
|
251
|
+
if (log->stored == MAX_BLOCK) {
|
252
|
+
(void)lseek(log->fd, log->last_off, SEEK_SET);
|
253
|
+
temp[0] = 0;
|
254
|
+
dice_ulg(log->stored + ((unsigned long)(~log->stored) << 16),
|
255
|
+
temp + 1);
|
256
|
+
if (write(log->fd, temp, 5) != 5)
|
257
|
+
return 1;
|
258
|
+
log->last_off = lseek(log->fd, log->stored, SEEK_CUR);
|
259
|
+
(void)lseek(log->fd, 5, SEEK_CUR);
|
260
|
+
log->stored = 0;
|
261
|
+
}
|
262
|
+
} while (len);
|
263
|
+
return 0;
|
264
|
+
}
|
265
|
+
|
266
|
+
/* recompress the remaining stored deflate data in place */
|
267
|
+
local int recomp(gz_log *log)
|
268
|
+
{
|
269
|
+
z_stream strm;
|
270
|
+
size_t len, max;
|
271
|
+
unsigned char *in;
|
272
|
+
unsigned char *out;
|
273
|
+
unsigned char temp[16];
|
274
|
+
|
275
|
+
/* allocate space and read it all in (it's around 1 MB) */
|
276
|
+
len = log->last_off - log->mark_off;
|
277
|
+
max = len + (len >> 12) + (len >> 14) + 11;
|
278
|
+
out = malloc(max);
|
279
|
+
if (out == NULL)
|
280
|
+
return 1;
|
281
|
+
in = malloc(len);
|
282
|
+
if (in == NULL) {
|
283
|
+
free(out);
|
284
|
+
return 1;
|
285
|
+
}
|
286
|
+
(void)lseek(log->fd, log->mark_off, SEEK_SET);
|
287
|
+
if (read(log->fd, in, len) != len) {
|
288
|
+
free(in);
|
289
|
+
free(out);
|
290
|
+
return 1;
|
291
|
+
}
|
292
|
+
|
293
|
+
/* recompress in memory, decoding stored data as we go */
|
294
|
+
/* note: this assumes that unsigned is four bytes or more */
|
295
|
+
/* consider not making that assumption */
|
296
|
+
strm.zalloc = Z_NULL;
|
297
|
+
strm.zfree = Z_NULL;
|
298
|
+
strm.opaque = Z_NULL;
|
299
|
+
if (deflateInit2(&strm, Z_BEST_COMPRESSION, Z_DEFLATED, -15, 8,
|
300
|
+
Z_DEFAULT_STRATEGY) != Z_OK) {
|
301
|
+
free(in);
|
302
|
+
free(out);
|
303
|
+
return 1;
|
304
|
+
}
|
305
|
+
strm.next_in = in;
|
306
|
+
strm.avail_out = max;
|
307
|
+
strm.next_out = out;
|
308
|
+
while (len >= 5) {
|
309
|
+
if (strm.next_in[0] != 0)
|
310
|
+
break;
|
311
|
+
strm.avail_in = strm.next_in[1] + (strm.next_in[2] << 8);
|
312
|
+
strm.next_in += 5;
|
313
|
+
len -= 5;
|
314
|
+
if (strm.avail_in != 0) {
|
315
|
+
if (len < strm.avail_in)
|
316
|
+
break;
|
317
|
+
len -= strm.avail_in;
|
318
|
+
(void)deflate(&strm, Z_NO_FLUSH);
|
319
|
+
if (strm.avail_in != 0 || strm.avail_out == 0)
|
320
|
+
break;
|
321
|
+
}
|
322
|
+
}
|
323
|
+
(void)deflate(&strm, Z_SYNC_FLUSH);
|
324
|
+
(void)deflateEnd(&strm);
|
325
|
+
free(in);
|
326
|
+
if (len != 0 || strm.avail_out == 0) {
|
327
|
+
free(out);
|
328
|
+
return 1;
|
329
|
+
}
|
330
|
+
|
331
|
+
/* overwrite stored data with compressed data */
|
332
|
+
(void)lseek(log->fd, log->mark_off, SEEK_SET);
|
333
|
+
len = max - strm.avail_out;
|
334
|
+
if (write(log->fd, out, len) != len) {
|
335
|
+
free(out);
|
336
|
+
return 1;
|
337
|
+
}
|
338
|
+
free(out);
|
339
|
+
|
340
|
+
/* write last empty block, crc, and length */
|
341
|
+
log->mark_off = log->last_off = lseek(log->fd, 0, SEEK_CUR);
|
342
|
+
temp[0] = 1;
|
343
|
+
dice_ulg(0xffffL << 16, temp + 1);
|
344
|
+
dice_ulg(log->crc, temp + 5);
|
345
|
+
dice_ulg(log->len, temp + 9);
|
346
|
+
if (write(log->fd, temp, 13) != 13)
|
347
|
+
return 1;
|
348
|
+
|
349
|
+
/* truncate file to discard remaining stored data and old trailer */
|
350
|
+
ftruncate(log->fd, lseek(log->fd, 0, SEEK_CUR));
|
351
|
+
|
352
|
+
/* update extra field to point to new last empty block */
|
353
|
+
(void)lseek(log->fd, log->extra, SEEK_SET);
|
354
|
+
dice_off(log->mark_off, temp);
|
355
|
+
dice_off(log->last_off, temp + 8);
|
356
|
+
if (write(log->fd, temp, 16) != 16)
|
357
|
+
return 1;
|
358
|
+
return 0;
|
359
|
+
}
|
360
|
+
|
361
|
+
/* maximum accumulation of stored blocks before compressing */
|
362
|
+
#define MAX_STORED 1048576
|
363
|
+
|
364
|
+
/* close log object */
|
365
|
+
int gzlog_close(void *obj)
|
366
|
+
{
|
367
|
+
unsigned char temp[8];
|
368
|
+
gz_log *log;
|
369
|
+
|
370
|
+
/* check object */
|
371
|
+
log = (gz_log *)obj;
|
372
|
+
if (log == NULL || log->id != GZLOGID)
|
373
|
+
return 1;
|
374
|
+
|
375
|
+
/* go to start of most recent block being written */
|
376
|
+
(void)lseek(log->fd, log->last_off, SEEK_SET);
|
377
|
+
|
378
|
+
/* if some stuff was put there, update block */
|
379
|
+
if (log->stored) {
|
380
|
+
temp[0] = 0;
|
381
|
+
dice_ulg(log->stored + ((unsigned long)(~log->stored) << 16),
|
382
|
+
temp + 1);
|
383
|
+
if (write(log->fd, temp, 5) != 5)
|
384
|
+
return 1;
|
385
|
+
log->last_off = lseek(log->fd, log->stored, SEEK_CUR);
|
386
|
+
}
|
387
|
+
|
388
|
+
/* write last block (empty) */
|
389
|
+
if (write(log->fd, "\001\000\000\377\377", 5) != 5)
|
390
|
+
return 1;
|
391
|
+
|
392
|
+
/* write updated crc and uncompressed length */
|
393
|
+
dice_ulg(log->crc, temp);
|
394
|
+
dice_ulg(log->len, temp + 4);
|
395
|
+
if (write(log->fd, temp, 8) != 8)
|
396
|
+
return 1;
|
397
|
+
|
398
|
+
/* put offset of that last block in gzip extra block */
|
399
|
+
(void)lseek(log->fd, log->extra + 8, SEEK_SET);
|
400
|
+
dice_off(log->last_off, temp);
|
401
|
+
if (write(log->fd, temp, 8) != 8)
|
402
|
+
return 1;
|
403
|
+
|
404
|
+
/* if more than 1 MB stored, then time to compress it */
|
405
|
+
if (log->last_off - log->mark_off > MAX_STORED) {
|
406
|
+
if (recomp(log))
|
407
|
+
return 1;
|
408
|
+
}
|
409
|
+
|
410
|
+
/* unlock and close file */
|
411
|
+
log_clean(log);
|
412
|
+
return 0;
|
413
|
+
}
|