libsql 0.1.0

Sign up to get free protection for your applications and to get access to all the features.
Files changed (98) hide show
  1. checksums.yaml +7 -0
  2. data/CONTRIBUTING.md +60 -0
  3. data/HISTORY.md +6 -0
  4. data/LICENSE +31 -0
  5. data/Manifest.txt +96 -0
  6. data/README.md +59 -0
  7. data/Rakefile +28 -0
  8. data/TODO.md +57 -0
  9. data/examples/a.rb +9 -0
  10. data/examples/blob.rb +106 -0
  11. data/examples/define_aggregate.rb +75 -0
  12. data/examples/define_function.rb +104 -0
  13. data/examples/fts5.rb +152 -0
  14. data/examples/gem-db.rb +94 -0
  15. data/examples/schema-info.rb +34 -0
  16. data/ext/libsql/c/extconf.rb +86 -0
  17. data/ext/libsql/c/gen_constants.rb +353 -0
  18. data/ext/libsql/c/libsql_blob.c +240 -0
  19. data/ext/libsql/c/libsql_constants.c +1518 -0
  20. data/ext/libsql/c/libsql_database.c +1188 -0
  21. data/ext/libsql/c/libsql_ext.c +383 -0
  22. data/ext/libsql/c/libsql_ext.h +149 -0
  23. data/ext/libsql/c/libsql_statement.c +649 -0
  24. data/ext/libsql/c/notes.txt +134 -0
  25. data/ext/libsql/c/sqlite3.c +247030 -0
  26. data/ext/libsql/c/sqlite3.h +13436 -0
  27. data/lib/libsql/aggregate.rb +73 -0
  28. data/lib/libsql/blob.rb +186 -0
  29. data/lib/libsql/boolean.rb +42 -0
  30. data/lib/libsql/busy_timeout.rb +47 -0
  31. data/lib/libsql/column.rb +99 -0
  32. data/lib/libsql/csv_table_importer.rb +75 -0
  33. data/lib/libsql/database.rb +933 -0
  34. data/lib/libsql/function.rb +61 -0
  35. data/lib/libsql/index.rb +43 -0
  36. data/lib/libsql/memory_database.rb +15 -0
  37. data/lib/libsql/paths.rb +80 -0
  38. data/lib/libsql/profile_tap.rb +131 -0
  39. data/lib/libsql/progress_handler.rb +21 -0
  40. data/lib/libsql/schema.rb +225 -0
  41. data/lib/libsql/sqlite3/constants.rb +95 -0
  42. data/lib/libsql/sqlite3/database/function.rb +48 -0
  43. data/lib/libsql/sqlite3/database/status.rb +68 -0
  44. data/lib/libsql/sqlite3/libsql_version.rb +32 -0
  45. data/lib/libsql/sqlite3/status.rb +60 -0
  46. data/lib/libsql/sqlite3/version.rb +55 -0
  47. data/lib/libsql/sqlite3.rb +7 -0
  48. data/lib/libsql/statement.rb +421 -0
  49. data/lib/libsql/table.rb +91 -0
  50. data/lib/libsql/taps/console.rb +27 -0
  51. data/lib/libsql/taps/io.rb +74 -0
  52. data/lib/libsql/taps.rb +2 -0
  53. data/lib/libsql/trace_tap.rb +35 -0
  54. data/lib/libsql/type_map.rb +63 -0
  55. data/lib/libsql/type_maps/default_map.rb +166 -0
  56. data/lib/libsql/type_maps/storage_map.rb +38 -0
  57. data/lib/libsql/type_maps/text_map.rb +21 -0
  58. data/lib/libsql/version.rb +8 -0
  59. data/lib/libsql/view.rb +26 -0
  60. data/lib/libsql-ruby.rb +1 -0
  61. data/lib/libsql.rb +51 -0
  62. data/spec/aggregate_spec.rb +158 -0
  63. data/spec/blob_spec.rb +78 -0
  64. data/spec/boolean_spec.rb +24 -0
  65. data/spec/busy_handler.rb +157 -0
  66. data/spec/data/iso-3166-country.txt +242 -0
  67. data/spec/data/iso-3166-schema.sql +22 -0
  68. data/spec/data/iso-3166-subcountry.txt +3995 -0
  69. data/spec/data/make-iso-db.sh +12 -0
  70. data/spec/database_spec.rb +505 -0
  71. data/spec/default_map_spec.rb +92 -0
  72. data/spec/function_spec.rb +78 -0
  73. data/spec/integeration_spec.rb +97 -0
  74. data/spec/iso_3166_database.rb +58 -0
  75. data/spec/json_spec.rb +24 -0
  76. data/spec/libsql_spec.rb +4 -0
  77. data/spec/paths_spec.rb +28 -0
  78. data/spec/progress_handler_spec.rb +91 -0
  79. data/spec/rtree_spec.rb +66 -0
  80. data/spec/schema_spec.rb +131 -0
  81. data/spec/spec_helper.rb +48 -0
  82. data/spec/sqlite3/constants_spec.rb +108 -0
  83. data/spec/sqlite3/database_status_spec.rb +36 -0
  84. data/spec/sqlite3/libsql_version_spec.rb +16 -0
  85. data/spec/sqlite3/status_spec.rb +22 -0
  86. data/spec/sqlite3/version_spec.rb +28 -0
  87. data/spec/sqlite3_spec.rb +53 -0
  88. data/spec/statement_spec.rb +168 -0
  89. data/spec/storage_map_spec.rb +38 -0
  90. data/spec/tap_spec.rb +57 -0
  91. data/spec/text_map_spec.rb +20 -0
  92. data/spec/type_map_spec.rb +14 -0
  93. data/spec/version_spec.rb +8 -0
  94. data/tasks/custom.rake +134 -0
  95. data/tasks/default.rake +257 -0
  96. data/tasks/extension.rake +29 -0
  97. data/tasks/this.rb +208 -0
  98. metadata +325 -0
data/examples/fts5.rb ADDED
@@ -0,0 +1,152 @@
1
+ #!/usr/bin/env ruby
2
+ require 'rubygems'
3
+ require 'libsql'
4
+ require 'benchmark'
5
+ require 'pathname'
6
+
7
+ begin
8
+ require 'json'
9
+ rescue LoadError
10
+ abort "'gem install json' to run this example"
11
+ end
12
+
13
+
14
+ README = <<_
15
+ This example program assumes that you have available the 'fortune' BSD command dataset.
16
+
17
+ execute `fortune -f` to see if you have this command available.
18
+ _
19
+
20
+ fortune_dir = %x[ fortune -f 2>&1 ].split("\n").first.split.last
21
+ abort README unless fortune_dir and File.directory?( fortune_dir )
22
+
23
+ fortune_path = Pathname.new(fortune_dir)
24
+
25
+ #
26
+ # Lets create a database that utilizes FTS5 http://www.sqlite.org/fts5.html
27
+ #
28
+ #
29
+
30
+ #
31
+ # Create a database, this will create the DB if it doesn't exist
32
+ #
33
+ puts "Opening database (version #{::Libsql::VERSION})"
34
+ db = ::Libsql::Database.new("fts5.db")
35
+
36
+ #
37
+ # Create the schema unless it already exists in the table. The meta information
38
+ # about the database schema is available as the result of the db.schema method
39
+ #
40
+ schema = db.schema
41
+ unless schema.tables['search']
42
+ puts "Create schema"
43
+ db.execute_batch <<-SQL
44
+ CREATE VIRTUAL TABLE search USING fts5(
45
+ filename,
46
+ content
47
+ );
48
+
49
+ CREATE TABLE plain (
50
+ filename VARCHAR(128),
51
+ content TEXT
52
+ );
53
+ SQL
54
+ db.reload_schema!
55
+ end
56
+
57
+ def each_fortune(path,&block)
58
+ fortune = []
59
+ path.each_line do |line|
60
+ line.strip!
61
+ if line == "%" then
62
+ yield fortune.join("\n")
63
+ fortune.clear
64
+ else
65
+ fortune << line
66
+ end
67
+ end
68
+ end
69
+
70
+ #
71
+ # Only load the data if the db is empty
72
+ #
73
+ if db.first_value_from( "SELECT count(*) from search" ) == 0 then
74
+ before = Time.now
75
+ idx = 0
76
+
77
+ # Inserting bulk rows as a transaction is good practice with SQLite, it is
78
+ # MUCH faster.
79
+ db.transaction do |db_in_transaction|
80
+ # Iterate over the files in the fortunes dir and split on the fortunes, then
81
+
82
+ fortune_path.each_child do |fortune_file|
83
+ next if fortune_file.directory?
84
+ next if fortune_file.extname == ".dat"
85
+ $stdout.puts "Loading #{fortune_file}"
86
+
87
+ each_fortune(fortune_file) do |fortune|
88
+ insert_data = {
89
+ ':fname' => fortune_file.to_s,
90
+ ':content' => fortune
91
+ }
92
+
93
+ # insert into the FTS5 table
94
+ db_in_transaction.prepare("INSERT INTO search( filename, content ) VALUES( :fname, :content );") do |stmt|
95
+ stmt.execute( insert_data )
96
+ end
97
+
98
+ # insert into the normal table for comparison
99
+ db_in_transaction.prepare("INSERT INTO plain( filename, content ) VALUES( :fname, :content );") do |stmt|
100
+ stmt.execute( insert_data )
101
+ end
102
+
103
+ idx += 1
104
+ print "Processed #{idx}\r"
105
+ $stdout.flush
106
+ end
107
+ puts "\nFinalizing..."
108
+ end
109
+ end
110
+ puts "Took #{Time.now - before} seconds to insert #{idx} fortunes"
111
+ puts "Done Inserting"
112
+ end
113
+
114
+ doc_count = db.first_value_from( "SELECT count(*) from search" )
115
+
116
+ #
117
+ # Now lets do some searching for some various words
118
+ #
119
+
120
+ %w[ president salmon thanks ].each do |word|
121
+
122
+ count = 100
123
+ puts
124
+ puts "Searching for '#{word}' #{count} times across #{doc_count} fortunes"
125
+ puts "=" * 60
126
+
127
+ Benchmark.bm( 15 ) do |x|
128
+
129
+ #
130
+ # search using the fts search to get the cont of tweets with the given word
131
+ #
132
+ x.report('fts5: ') do
133
+ db.prepare( "SELECT count(filename) FROM search WHERE search MATCH 'content:#{word}'" ) do |stmt|
134
+ count.times do
135
+ stmt.execute
136
+ end
137
+ end
138
+ end
139
+
140
+ #
141
+ # search using basic string matching in sqlite.
142
+ #
143
+ x.report('plain: ') do
144
+ db.prepare( "SELECT count(filename) FROM plain WHERE content LIKE '% #{word} %'" ) do |stmt|
145
+ count.times do
146
+ stmt.execute
147
+ end
148
+ end
149
+ end
150
+ end
151
+ end
152
+
@@ -0,0 +1,94 @@
1
+ #!/usr/bin/env ruby
2
+
3
+ #
4
+ # Basic libsql example creating a table, inserting rows and doing various
5
+ # selects and prepared statements
6
+ #
7
+ require 'rubygems'
8
+ require 'libsql'
9
+
10
+ #
11
+ # Create a database, this will create the DB if it doesn't exist
12
+ #
13
+ puts "Opening database (version #{::Libsql::Version})"
14
+ db = ::Libsql::Database.new("gems.db")
15
+
16
+ #
17
+ # Setup taps into profile and trace information of sqlite, the profile tap will
18
+ # goto the profile_tap.log file and the trace information will go to the
19
+ # trace_tap.log file
20
+ #
21
+ puts "Establishing taps"
22
+ db.trace_tap = ::Libsql::Taps::IO.new( trace_tap_file = File.open("trace_tap.log", "w+") )
23
+ db.profile_tap = ::Libsql::Taps::IO.new( profile_tap_file = File.open("profile_tap.log", "w+") )
24
+
25
+ #
26
+ # Create the schema unless it already exists in the table. The meta information
27
+ # about the database schema is available as the result of the db.schema method
28
+ #
29
+ schema = db.schema
30
+ unless schema.tables['gems']
31
+ puts "Create schema"
32
+ db.execute <<-SQL
33
+ CREATE TABLE gems (
34
+ id INTEGER NOT NULL PRIMARY KEY AUTOINCREMENT,
35
+ name VARCHAR(128),
36
+ version VARCHAR(32),
37
+ author VARCHAR(128)
38
+ );
39
+ SQL
40
+ db.reload_schema!
41
+ end
42
+
43
+ #
44
+ # Get some data from the system to insert into the database. Since everyone
45
+ # probably has gems installed, that's a ready known piece of information. We'll
46
+ # just pull in the latest version of each installed gem and dump some meta
47
+ # information into a db for testing.
48
+ #
49
+ latest_specs = Gem.source_index.latest_specs
50
+
51
+ puts "Inserting #{latest_specs.size} rows of gem information..."
52
+ before = Time.now
53
+
54
+ # Inserting bulk rows as a transaction is good practice with SQLite, it is
55
+ # MUCH faster.
56
+ db.transaction do |db_in_transaction|
57
+ db_in_transaction.prepare("INSERT INTO gems(name, version, author) VALUES( :name, :version, :author );") do |stmt|
58
+ latest_specs.each do |spec|
59
+ insert_data = {}
60
+ insert_data[':name'] = spec.name.to_s
61
+ insert_data[':version'] = spec.version.to_s
62
+ insert_data[':author'] = spec.authors.join(' ')
63
+ #puts "Inserting #{insert_data.inspect}"
64
+ stmt.execute( insert_data )
65
+ end
66
+ end
67
+ end
68
+ puts "Took #{Time.now - before} seconds"
69
+ puts "Done Inserting"
70
+
71
+ authors_by_number = db.execute("SELECT author, count( name ) as num_gems FROM gems GROUP BY author ORDER BY num_gems DESC")
72
+ favorite_author = authors_by_number.first
73
+ puts "Your favorite gem author is <#{favorite_author['author']}>, with #{favorite_author['num_gems']} gems installed."
74
+
75
+ #
76
+ # Now we'll look at the profile sampler and see what information it traced about
77
+ # our behavoir.
78
+ #
79
+ db.profile_tap.samplers.each do |stat_name, stat_values|
80
+ puts "-" * 20
81
+ puts stat_values.to_s
82
+ end
83
+
84
+ #
85
+ # Clear out the taps (not really necessary, just cleaning up)
86
+ #
87
+ db.trace_tap = profile_tap = nil
88
+
89
+ #
90
+ # close things down
91
+ #
92
+ db.close
93
+ trace_tap_file.close
94
+ profile_tap_file.close
@@ -0,0 +1,34 @@
1
+ #!/usr/bin/env ruby
2
+
3
+ require 'rubygems'
4
+ require 'libsql'
5
+
6
+ db_name = ARGV.shift
7
+ unless db_name
8
+ puts "Usage: #{File.basename($0)} dbname"
9
+ exit 1
10
+ end
11
+ db = ::Libsql::Database.new( db_name )
12
+ col_info = %w[ default_value declared_data_type collation_sequence_name not_null_constraint primary_key auto_increment ]
13
+ max_width = col_info.collect { |c| c.length }.sort.last
14
+
15
+ db.schema.tables.keys.sort.each do |table_name|
16
+ puts "Table: #{table_name}"
17
+ puts "=" * 42
18
+ db.schema.tables[table_name].columns.each_pair do |col_name, col|
19
+ puts " Column : #{col.name}"
20
+ col_info.each do |ci|
21
+ puts " |#{ci.rjust( max_width, "." )} : #{col.send( ci )}"
22
+ end
23
+ puts
24
+ end
25
+
26
+ db.schema.tables[table_name].indexes.each_pair do |idx_name, index|
27
+ puts " Index : #{index.name}"
28
+ puts " |#{"sequence_number".rjust( max_width, "." )} : #{index.sequence_number}"
29
+ puts " |#{"is unique?".rjust( max_width, ".")} : #{index.unique?}"
30
+ puts " |#{"columns".rjust( max_width, ".")} : #{index.columns.collect { |c| c.name }.join(',') }"
31
+ puts
32
+ end
33
+ end
34
+
@@ -0,0 +1,86 @@
1
+ require 'mkmf'
2
+ require 'rbconfig'
3
+
4
+ # used by the ext:build_win-1.x.x tasks, really no one else but jeremy should be
5
+ # using this hack
6
+ $ruby = ARGV.shift if ARGV[0]
7
+
8
+ # make available table and column meta data api
9
+ $CFLAGS += " -DSQLITE_ENABLE_BYTECODE_VTAB=1"
10
+ $CFLAGS += " -DSQLITE_ENABLE_COLUMN_METADATA=1"
11
+ $CFLAGS += " -DSQLITE_ENABLE_DBSTAT_VTAB=1"
12
+ $CFLAGS += " -DSQLITE_ENABLE_DBPAGE_VTAB=1"
13
+ $CFLAGS += " -DSQLITE_ENABLE_EXPLAIN_COMMENTS=1"
14
+ $CFLAGS += " -DSQLITE_ENABLE_FTS3=1"
15
+ $CFLAGS += " -DSQLITE_ENABLE_FTS3_PARENTHESIS=1"
16
+ $CFLAGS += " -DSQLITE_ENABLE_FTS4=1"
17
+ $CFLAGS += " -DSQLITE_ENABLE_FTS5=1"
18
+ $CFLAGS += " -DSQLITE_ENABLE_GEOPOLY=1"
19
+ $CFLAGS += " -DSQLITE_ENABLE_MATH_FUNCTIONS=1"
20
+ $CFLAGS += " -DSQLITE_ENABLE_MEMORY_MANAGEMENT=1"
21
+ $CFLAGS += " -DSQLITE_ENABLE_NORMALIZE=1"
22
+ $CFLAGS += " -DSQLITE_ENABLE_NULL_TRIM=1"
23
+ $CFLAGS += " -DSQLITE_ENABLE_PREUPDATE_HOOK=1"
24
+ $CFLAGS ++ " -DSQLITE_EANBLE_QPSG=1"
25
+ $CFLAGS += " -DSQLITE_ENABLE_RBU=1"
26
+ $CFLAGS += " -DSQLITE_ENABLE_RTREE=1"
27
+ $CFLAGS += " -DSQLITE_ENABLE_SESSION=1"
28
+ # https://github.com/libsql/libsql/issues/144
29
+ # $CFLAGS += " -DSQLITE_ENABLE_SNAPSHOT=1"
30
+ $CFLAGS += " -DSQLITE_ENABLE_STMTVTAB=1"
31
+ $CFLAGS += " -DSQLITE_ENABLE_STAT4=1"
32
+ $CFLAGS += " -DSQLITE_ENABLE_UNLOCK_NOTIFY=1"
33
+ $CFLAGS += " -DSQLITE_ENABLE_SOUNDEX=1"
34
+
35
+ $CFLAGS += " -DSQLITE_USE_ALLOCA=1"
36
+ $CFLAGS += " -DSQLITE_OMIT_DEPRECATED=1"
37
+
38
+ # we compile sqlite the same way that the installation of ruby is compiled.
39
+ if RbConfig::MAKEFILE_CONFIG['configure_args'].include?( "--enable-pthread" ) then
40
+ $CFLAGS += " -DSQLITE_THREADSAFE=1"
41
+ else
42
+ $CFLAGS += " -DSQLITE_THREADSAFE=0"
43
+ end
44
+
45
+ # remove the -g flags if it exists
46
+ %w[ -ggdb\\d* -g\\d* ].each do |debug|
47
+ $CFLAGS = $CFLAGS.gsub(/\s#{debug}\b/,'')
48
+ RbConfig::MAKEFILE_CONFIG['debugflags'] = RbConfig::MAKEFILE_CONFIG['debugflags'].gsub(/\s#{debug}\b/,'') if RbConfig::MAKEFILE_CONFIG['debugflags']
49
+ end
50
+
51
+ ignoreable_warnings = %w[ write-strings ]
52
+ ignore_by_compiler = {
53
+ "clang" => %w[
54
+ empty-body
55
+ declaration-after-statement
56
+ incompatible-pointer-types-discards-qualifiers
57
+ shorten-64-to-32
58
+ sign-compare
59
+ unused-const-variable
60
+ unused-variable
61
+ unused-but-set-variable
62
+ undef
63
+ ],
64
+ "gcc" => %w[
65
+ declaration-after-statement
66
+ implicit-function-declaration
67
+ unused-variable
68
+ unused-but-set-variable
69
+ maybe-uninitialized
70
+ old-style-definition
71
+ undef
72
+ ]
73
+ }
74
+
75
+ if extras = ignore_by_compiler[RbConfig::MAKEFILE_CONFIG["CC"]] then
76
+ ignoreable_warnings.concat(extras)
77
+ end
78
+
79
+ ignoreable_warnings.each do |warning|
80
+ $CFLAGS = $CFLAGS.gsub(/-W#{warning}/,'')
81
+ RbConfig::MAKEFILE_CONFIG['warnflags'] = RbConfig::MAKEFILE_CONFIG['warnflags'].gsub(/-W#{warning}/,'') if RbConfig::MAKEFILE_CONFIG['warnflags']
82
+ $CFLAGS += " -Wno-#{warning}"
83
+ end
84
+
85
+ subdir = RUBY_VERSION.sub(/\.\d$/,'')
86
+ create_makefile("libsql/#{subdir}/libsql_ext")