heroku-tokyotyrant 0.4

data/ext/tokyocabinet-1.4.41/f.tsv

@@ -0,0 +1,2 @@
+10341718880075105600	datestr	2009.12.01 07:27:37	from	postmaster@nomail.com (postmaster)	subject	Attx PostMaster x Tokyo Cabinets x Fast Index In x2009x x14x17x x xdatastore.vol01x02xTAR2x	groupid	64	maxpartno	4
+9105636605948918107	datestr	2009.12.01 07:27:40	from	postmaster@nomail.com (postmaster)	subject	Attx PostMaster x Tokyo Cabinets x Fast Index In x2009x x10x17x x x11 x Tokyo Cabinets x No Replacementxdirx	groupid	64	maxpartno	45

data/ext/tokyocabinet-1.4.41/lab/calccomp

@@ -0,0 +1,118 @@
+#! /usr/bin/perl
+
+#================================================================
+# calccomp
+# Measure elapsed time and database size of compression algos
+#================================================================
+
+use strict;
+use warnings;
+use Time::HiRes qw(gettimeofday);
+
+use constant {
+    WORDFILE => '/usr/share/dict/words',
+    TSVFILE => 'words.tsv',
+    NULLFILE => '/dev/null',
+    BNUM => 262144,
+    LCNUM => 8192,
+    TESTCOUNT => 20,
+    REMOVETOP => 2,
+    REMOVEBOTTOM => 8,
+};
+
+open(IN, '<' . WORDFILE) or die(WORDFILE . ' could not be opened');
+open(OUT, '>' . TSVFILE) or die(TSVFILE . ' could not be opened');
+while(defined(my $word = <IN>)){
+    $word =~ s/[\t\r\n]//g;
+    next if(length($word) < 1);
+    printf OUT ("%s\t%d\n", $word, int(rand(10000)));
+}
+close(OUT);
+close(IN);
+
+my @commands = (
+    './tchmgr create casket-hash ' . BNUM . ' ; ./tchmgr importtsv casket-hash ' . TSVFILE,
+    './tchmgr list casket-hash >' . NULLFILE,
+    './tcbmgr create casket-btree ' . LCNUM . ' ; ./tcbmgr importtsv casket-btree ' . TSVFILE,
+    './tcbmgr list casket-btree >' . NULLFILE,
+    './tcbmgr create -td casket-bt-td ' . LCNUM . ' ; ./tcbmgr importtsv casket-bt-td ' . TSVFILE,
+    './tcbmgr list casket-bt-td >' . NULLFILE,
+    './tcbmgr create -tb casket-bt-tb ' . LCNUM . ' ; ./tcbmgr importtsv casket-bt-tb ' . TSVFILE,
+    './tcbmgr list casket-bt-tb >' . NULLFILE,
+    './tcbmgr create -tt casket-bt-tt ' . LCNUM . ' ; ./tcbmgr importtsv casket-bt-tt ' . TSVFILE,
+    './tcbmgr list casket-bt-tt >' . NULLFILE,
+    './tcbmgr create -tx casket-bt-tx ' . LCNUM . ' ; ./tcbmgr importtsv casket-bt-tx ' . TSVFILE,
+    './tcbmgr list casket-bt-tx >' . NULLFILE,
+    );
+
+my @names = (
+    'casket-hash',
+    'casket-btree',
+    'casket-bt-td',
+    'casket-bt-tb',
+    'casket-bt-tt',
+    'casket-bt-tx',
+    );
+
+foreach my $name (@names){
+    my @paths = glob("$name*");
+    foreach my $path (@paths){
+        unlink($path);
+    }
+}
+
+my @table;
+foreach my $command (@commands){
+    system('sync ; sync');
+    $ENV{'HIDEPRGR'} = 1;
+    my @result;
+    for(my $i = 0; $i < TESTCOUNT; $i++){
+        my $stime = gettimeofday();
+        system("$command >/dev/null 2>&1");
+        $stime = gettimeofday() - $stime;
+        printf("%s\t%d\t%0.5f\n", $command, $i + 1, $stime);
+        push(@result, $stime);
+    }
+    @result = sort { $a <=> $b } @result;
+    for(my $i = 0; $i < REMOVETOP; $i++){
+        shift(@result);
+    }
+    for(my $i = 0; $i < REMOVEBOTTOM; $i++){
+        pop(@result);
+    }
+    my $sum = 0;
+    foreach my $result (@result){
+        $sum += $result;
+    }
+    my $avg = $sum / scalar(@result);
+    push(@table, [$command, $avg]);
+}
+
+printf("\n\nRESULT\n\n");
+foreach my $row (@table){
+    printf("%s\t%0.5f\n", $$row[0], $$row[1]);
+}
+printf("\n");
+
+my @sizes;
+foreach my $name (@names){
+    my @sbuf = stat($name);
+    my $size = $sbuf[7];
+    printf("%s\t%s\n", $name, $size);
+    push(@sizes, $size);
+}
+my @sbuf = stat(TSVFILE);
+my $size = $sbuf[7];
+printf("\n");
+
+printf("%s,%.5f,%.5f,%d\n", "ORIGINAL", 0, 0, $size);
+printf("%s,%.5f,%.5f,%d\n", "HASH", $table[0][1], $table[1][1], $sizes[0]);
+printf("%s,%.5f,%.5f,%d\n", "BTREE", $table[2][1], $table[3][1], $sizes[1]);
+printf("%s,%.5f,%.5f,%d\n", "BTREE-DEFLATE", $table[4][1], $table[5][1], $sizes[2]);
+printf("%s,%.5f,%.5f,%d\n", "BTREE-BZIP2", $table[6][1], $table[7][1], $sizes[3]);
+printf("%s,%.5f,%.5f,%d\n", "BTREE-TCBS", $table[8][1], $table[9][1], $sizes[4]);
+printf("%s,%.5f,%.5f,%d\n", "BTREE-EXCODEC", $table[10][1], $table[11][1], $sizes[5]);
+
+
+
+# END OF FILE

data/ext/tokyocabinet-1.4.41/lab/datechange

@@ -0,0 +1,56 @@
+#! /bin/sh
+
+#================================================================
+# datechange
+# Replace date expressions
+#================================================================
+
+
+# set variables
+LANG=C
+LC_ALL=C
+export LANG LC_ALL
+year=`date '+%Y'`
+date=`date '+%Y-%m-%d'`
+fulldate=`date -R`
+regexsrc='\.(h|c|cc|cpp|cxx|java|pl|pm|pod|rb|rd|lua)$'
+regexman='\.[0-9]$'
+regexhtml='\.html$'
+
+
+# edit source files
+find . -type f | egrep "$regexsrc" |
+while read file
+do
+  echo "$file"
+  sed "/opyright/ s/\\(20[0-9][0-9]\\)-\\(20[0-9][0-9]\\)/\\1-$year/" "$file" > "$file.tmp"
+  mv -f "$file.tmp" "$file"
+done
+
+
+# edit manual files
+find . -type f | egrep "$regexman" |
+while read file
+do
+  echo "$file"
+  sed "/\\.TH/ s/\\(20[0-9][0-9]\\)-\\([0-9][0-9]\\)-\\([0-9][0-9]\\)/$date/" "$file" > "$file.tmp"
+  mv -f "$file.tmp" "$file"
+done
+
+
+# edit HTML files
+find . -type f | egrep "$regexhtml" |
+while read file
+do
+  echo "$file"
+  sed -e "/opyright/ s/\\(20[0-9][0-9]\\)-\\(20[0-9][0-9]\\)/\\1-$year/" -e "/<div class=\"note\">/ s/Last Update: *\\([A-Z][a-z][a-z], [0-9][0-9] [A-Z][a-z][a-z] 20[0-9][0-9] [0-9][0-9]:[0-9][0-9]:[0-9][0-9] +[0-9][0-9]*\\)/Last Update: $fulldate/" "$file" > "$file.tmp"
+  mv -f "$file.tmp" "$file"
+done
+
+
+# exit normally
+exit 0
+
+
+
+# END OF FILE

data/ext/tokyocabinet-1.4.41/lab/diffcheck

@@ -0,0 +1,45 @@
+#! /bin/sh
+
+#================================================================
+# diffcheck
+# List files different from ones of another version
+#================================================================
+
+
+# set variables
+LANG=C
+LC_ALL=C
+export LANG LC_ALL
+regex='\.(h|c|cc|cpp|cxx|java|pl|pm|pod|rb|rd|lua|[1-9]|html|txt)$'
+
+
+# check arguments
+if [ $# != 1 ]
+then
+  printf 'diffcheck: usage: diffcheck directory_of_oldversion\n' 1>&2
+  exit 1
+fi
+
+
+# diff files
+find . -type f | egrep $regex |
+while read file
+do
+  old=`printf '%s\n' "$file" | sed 's/^\.\///'`
+  printf 'Checking %s and %s ... ' "$file" "$1/$old"
+  res=`diff -q "$file" "$1/$old"`
+  if [ -z "$res" ]
+  then
+    printf 'same\n'
+  else
+    printf '### !!! DIFFERENT !!! ###\n'
+  fi
+done
+
+
+# exit normally
+exit 0
+
+
+
+# END OF FILE

data/ext/tokyocabinet-1.4.41/lab/htmltotsv

@@ -0,0 +1,102 @@
+#! /usr/bin/perl
+
+#================================================================
+# htmldoctotsv
+# Generate TSV data from HTML documents
+#================================================================
+
+use strict;
+use warnings;
+use Encode;
+use Cwd 'realpath';
+
+my $path = '.';
+my $mode_kv = 0;
+my $mode_hex = 0;
+
+for(my $i = 0; $i < scalar(@ARGV); $i++){
+    my $arg = $ARGV[$i];
+    if($arg =~ /^-/){
+        if($arg eq '-kv'){
+            $mode_kv = 1;
+        } elsif($arg eq '-x'){
+            $mode_hex = 1;
+        }
+    } else {
+        $path = $arg;
+    }
+}
+
+sub trimhtml {
+    my $text = shift;
+    $text =~ s/<[^>]*>/ /g;
+    $text =~ s/&lt;/</g;
+    $text =~ s/&gt;/>/g;
+    $text =~ s/&quot;/"/g;
+    $text =~ s/&nbsp;/ /g;
+    $text =~ s/&amp;/\&/g;
+    $text =~ s/\s+/ /g;
+    $text =~ s/^ *//;
+    $text =~ s/ *$//;
+    return $text;
+}
+
+$ENV{LANG} = "C";
+$ENV{LC_ALL} = "C";
+open(my $lfh, "find $path -type f -iregex '.*\.html?' -print | sort |") || die("could not open");
+my $id = 0;
+while(defined($path = <$lfh>)){
+    chomp($path);
+    $path = realpath($path);
+    next if(!defined($path));
+    my @stat = stat($path);
+    next if(scalar(@stat) < 10);
+    my $mtime = $stat[9];
+    open(my $ifh, "<$path") || next;
+    my $encname = "UTF-8";
+    my @lines;
+    while(defined(my $line = <$ifh>)){
+        push(@lines, $line);
+        if($line =~ /<meta.*content-type.*charset=/i){
+            $line =~ s/.*charset=?//i;
+            $line =~ s/[^-_a-zA-Z0-9].*//;
+            chomp($line);
+            $encname = $line if(length($line) > 0);
+        }
+    }
+    my $text = join('', @lines);
+    $text = encode("UTF-8", decode($encname, $text)) if($encname ne "UTF-8");
+    $text =~ s/<!--.*?-->//is;
+    my $title = "";
+    if($text =~ /<title[^>]*>[^<]*<\/title>/i){
+        $title = $text;
+        $title =~ s/.*<title[^>]*>([^<]*)<\/title>.*/$1/is;
+        $title = trimhtml($title);
+    }
+    $text =~ s/.*<body[^>]*>(.*)<\/body>.*/$1/is;
+    $text =~ s/<style[^>]*>.*?<\/style>//is;
+    $text =~ s/<script[^>]*>.*?<\/script>//is;
+    $text = trimhtml($text);
+    next if(length($title) < 1 && length($text) < 1);
+    $id++;
+    my $key = $mode_hex ? sprintf("%X", $id) : $id;
+    printf STDERR ("%d: saving: %s\n", $id, $path);
+    if($mode_kv){
+        $text = $title . " " . $text if(length($title) > 0);
+        printf("%s\t%s\n", $key, $text);
+    } else {
+        printf("%s", $key);
+        printf("\turl\t%s", $path);
+        printf("\tsize\t%s", length($text));
+        printf("\tmtime\t%s", $mtime);
+        printf("\ttitle\t%s", $title) if(length($title) > 0);
+        printf("\tbody\t%s", $text);
+        printf("\n");
+    }
+    close($ifh);
+}
+close($lfh);
+
+
+
+# END OF FILE

data/ext/tokyocabinet-1.4.41/lab/magic

@@ -0,0 +1,19 @@
+# Tokyo Cabinet magic data
+0       string    ToKyO\ CaBiNeT\n   Tokyo Cabinet
+>14     string    x                  \b (%s)
+>32     byte      0                  \b, Hash
+>32     byte      1                  \b, B+ tree
+>32     byte      2                  \b, Fixed-length
+>32     byte      3                  \b, Table
+>33     byte      &1                 \b, [open]
+>33     byte      &2                 \b, [fatal]
+>34     byte      x                  \b, apow=%d
+>35     byte      x                  \b, fpow=%d
+>36     byte      &1                 \b, [large]
+>36     byte      &2                 \b, [deflate]
+>36     byte      &4                 \b, [bzip]
+>36     byte      &8                 \b, [tcbs]
+>36     byte      &16                \b, [excodec]
+>40     lelong    x                  \b, bnum=%d
+>48     lelong    x                  \b, rnum=%d
+>56     lelong    x                  \b, fsiz=%d

data/ext/tokyocabinet-1.4.41/lab/printenv.cgi

@@ -0,0 +1,27 @@
+#! /bin/sh
+
+#================================================================
+# printenv.cgi
+# Print CGI environment variables
+#================================================================
+
+
+# set variables
+LANG=C
+LC_ALL=C
+export LANG LC_ALL
+
+
+# output the result
+printf 'Content-Type: text/plain\r\n'
+printf '\r\n'
+
+printenv | sort
+
+
+# exit normally
+exit 0
+
+
+
+# END OF FILE

data/ext/tokyocabinet-1.4.41/lab/stepcount

@@ -0,0 +1,26 @@
+#! /bin/sh
+
+#================================================================
+# stepcount
+# Find files including dispensable tab characters
+#================================================================
+
+
+# set variables
+LANG=C
+LC_ALL=C
+export LANG LC_ALL
+regex='(\.h|\.c|\.cc|\.java|\.pl|\.pm|\.xs|\.rb|\.js)$'
+
+
+# count steps
+files=`find . -type f | egrep $regex`
+wc -l $files | sort -n
+
+
+# exit normally
+exit 0
+
+
+
+# END OF FILE

data/ext/tokyocabinet-1.4.41/lab/stopwatch

@@ -0,0 +1,61 @@
+#! /usr/bin/perl
+
+#================================================================
+# stopwatch
+# Measure elapsed time of some test commands.
+#================================================================
+
+use strict;
+use warnings;
+use Time::HiRes qw(gettimeofday);
+
+use constant {
+    TESTCOUNT => 20,
+    REMOVETOP => 2,
+    REMOVEBOTTOM => 8,
+};
+
+my @commands = (
+                "./tchtest write casket 1000000 1000000",
+                "./tchtest read casket",
+                "./tchtest rcat casket 1000000",
+                "./tcbtest write casket 1000000",
+                "./tcbtest read casket",
+                "./tcbtest rcat -lc 256 -nc 128 casket 100000",
+                );
+
+my @table;
+foreach my $command (@commands){
+    system("sync ; sync");
+    my @result;
+    for(my $i = 0; $i < TESTCOUNT; $i++){
+        my $stime = gettimeofday();
+        system("$command >/dev/null 2>&1");
+        $stime = gettimeofday() - $stime;
+        printf("%s\t%d\t%0.5f\n", $command, $i + 1, $stime);
+        push(@result, $stime);
+    }
+    @result = sort { $a <=> $b } @result;
+    for(my $i = 0; $i < REMOVETOP; $i++){
+        shift(@result);
+    }
+    for(my $i = 0; $i < REMOVEBOTTOM; $i++){
+        pop(@result);
+    }
+    my $sum = 0;
+    foreach my $result (@result){
+        $sum += $result;
+    }
+    my $avg = $sum / scalar(@result);
+    push(@table, [$command, $avg]);
+}
+
+printf("\n\nRESULT\n");
+foreach my $row (@table){
+    printf("%s\t%0.5f\n", $$row[0], $$row[1]);
+}
+printf("\n");
+
+
+
+# END OF FILE