amalgalite 1.0.0-x86-mswin32 → 1.1.1-x86-mswin32

Sign up to get free protection for your applications and to get access to all the features.
@@ -1,4 +1,20 @@
1
1
  = Amalgalite Changelog
2
+ == Version 1.1.1 - 2011-03-27
3
+
4
+ === Enhancements
5
+
6
+ * Update to SQLite 3.7.5
7
+ * Enable SQLite Full Text Search compile time options FTS3 and FTS4 [github issue #10]
8
+ http://www.sqlite.org/fts3.html
9
+ * Enable STAT2 SQLite compile time option to improve ANALYZE command support
10
+ http://www.sqlite.org/lang_analyze.html
11
+
12
+ === Bug Fixes
13
+
14
+ * The C coded generated by gen_constants.rb was updated to support older
15
+ compilers [github issue #8] (patch from josb)
16
+ * Fix 'amalgalite-pack --self' on ruby 1.9
17
+
2
18
  == Version 1.0.0 - 2011-01-16
3
19
 
4
20
  === Enhancements
@@ -1,7 +1,6 @@
1
1
  #!/usr/bin/env ruby
2
2
  require 'optparse'
3
3
  require 'pathname'
4
- require 'rubygems'
5
4
 
6
5
  #
7
6
  # add relative paths to the load path if we are not a gem and calculate what the
@@ -16,14 +15,36 @@ if ".." == rel_path.to_s.split( File::SEPARATOR ).first then
16
15
  $:.unshift File.join( File.dirname( __FILE__ ), "../ext" )
17
16
  end
18
17
 
19
- #
18
+ #
20
19
  # snapshot of what is needed for amalgalite requires, this info may only be used
21
20
  # when packing amalgalite itself
22
21
  #
23
22
  loaded_features_before = $LOADED_FEATURES.dup
24
23
  require 'amalgalite/packer'
25
24
  loaded_features_after = $LOADED_FEATURES.dup
26
- amalgalite_needs = loaded_features_after - loaded_features_before
25
+ load_diff = loaded_features_after - loaded_features_before
26
+
27
+ #
28
+ # strip off any LOAD_PATH elements from the front of load_diff since that
29
+ # will conflict with Amalgalite::Packer.amalgalite_require_order. Also
30
+ # strip out any 'rubygems' items since those are not used by Amalgalite
31
+ # and show as a side effect fo the "require 'amalgalite/packer'"
32
+ #
33
+ strip_paths = $LOAD_PATH.sort.reverse
34
+ amalgalite_needs = []
35
+ load_diff.each do |f|
36
+ next if f.split( File::SEPARATOR ).include?( "rubygems" )
37
+ appended = false
38
+ strip_paths.each do |path|
39
+ if 0 == f.index(path ) then
40
+ rel_path = f.sub( path, '' ).sub(%r{\A#{File::SEPARATOR}},'')
41
+ amalgalite_needs << rel_path
42
+ appended = true
43
+ break
44
+ end
45
+ end
46
+ amalgalite_needs << f unless appended
47
+ end
27
48
 
28
49
  #
29
50
  # Commandline parser
@@ -77,7 +98,7 @@ begin
77
98
  require 'amalgalite/packer'
78
99
  file_list = ARGV.dup
79
100
 
80
-
101
+
81
102
  if options[:self] then
82
103
  options[:table_name] = Amalgalite::Requires::Bootstrap::DEFAULT_BOOTSTRAP_TABLE
83
104
  core_libs = (amalgalite_needs - Amalgalite::Packer.amalgalite_require_order).delete_if { |l| l.index(".rb").nil? }
@@ -85,9 +106,9 @@ begin
85
106
  #
86
107
  # check and make sure nothing is missed
87
108
  #
88
- core_libs.each do |l|
89
- if l.index("malgalite") then
90
- STDERR.puts "ERROR! require_order needs an update #{l}"
109
+ core_libs.each do |l|
110
+ if l.index("amalgalite") then
111
+ STDERR.puts "ERROR! require_order needs an update #{l}"
91
112
  exit 2
92
113
  end
93
114
  end
@@ -116,7 +137,7 @@ Packing complete. To utilize the bootstrapping in #{dbfile} you must do
116
137
  one of the following:
117
138
 
118
139
  * statically compile the amalgalite C extension into your application
119
- * require 'amalgalite/amalgalite3'
140
+ * require 'amalgalite/#{RUBY_VERSION.sub(/\.\d$/,'')}/amalgalite3'
120
141
 
121
142
  Once one of those is working you can boostrap the Amalgalite library with
122
143
  this line in your code:
Binary file
@@ -0,0 +1,144 @@
1
+ #!/usr/bin/env ruby
2
+ require 'rubygems'
3
+ require 'amalgalite'
4
+ require 'benchmark'
5
+
6
+ begin
7
+ require 'json'
8
+ rescue LoadError
9
+ abort "'gem install json' to run this example"
10
+ end
11
+
12
+
13
+ README = <<_
14
+ This example programs assumes that you have downloaded the 'Tweets During the
15
+ State of the Union address' dataset from infochimps.com
16
+
17
+ http://www.infochimps.com/datasets/tweets-during-state-of-the-union-address
18
+
19
+ Please:
20
+ 1) download this dataset
21
+ 2) bunzip + untar this file
22
+ 3) record the location of the 'twitter_stream.txt' file that is in the
23
+ tarball.
24
+ 4) Pass this file as the first parameter to this script.
25
+ _
26
+
27
+ twitter_stream = ARGV.shift
28
+ abort README unless twitter_stream and File.readable?( twitter_stream )
29
+
30
+ #
31
+ # Lets create a database that utilizes FTS3 http://www.sqlite.org/fts3.html
32
+ #
33
+ #
34
+
35
+ #
36
+ # Create a database, this will create the DB if it doesn't exist
37
+ #
38
+ puts "Opening database (version #{Amalgalite::Version})"
39
+ db = Amalgalite::Database.new("fts3.db")
40
+
41
+ #
42
+ # Create the schema unless it already exists in the table. The meta information
43
+ # about the database schema is available as the result of the db.schema method
44
+ #
45
+ schema = db.schema
46
+ unless schema.tables['search']
47
+ puts "Create schema"
48
+ db.execute_batch <<-SQL
49
+ CREATE VIRTUAL TABLE search USING fts3(
50
+ filename VARCHAR(128),
51
+ content TEXT
52
+ );
53
+
54
+ CREATE TABLE plain (
55
+ filename VARCHAR(128),
56
+ content TEXT
57
+ );
58
+ SQL
59
+ db.reload_schema!
60
+ end
61
+
62
+ #
63
+ # Only load the data if the db is empty
64
+ #
65
+ if db.first_value_from( "SELECT count(*) from search" ) == 0 then
66
+ before = Time.now
67
+ #
68
+ # Load the tweets from the file passed on the commandline into the database
69
+ # We just want the text and the tweet id and insert that into the database.
70
+ #
71
+
72
+ lines = IO.readlines( twitter_stream )
73
+ idx = 0
74
+
75
+ # Inserting bulk rows as a transaction is good practice with SQLite, it is
76
+ # MUCH faster.
77
+ db.transaction do |db_in_transaction|
78
+ lines.each do |line|
79
+
80
+ # quick little parse of the tweet
81
+ json = JSON.parse( line )
82
+ insert_data = {}
83
+ insert_data[':fname'] = json['id']
84
+ insert_data[':content'] = json['text']
85
+
86
+ # insert into the FTS3 table
87
+ db_in_transaction.prepare("INSERT INTO search( filename, content ) VALUES( :fname, :content );") do |stmt|
88
+ stmt.execute( insert_data )
89
+ end
90
+
91
+ # insert into the normal table for comparison
92
+ db_in_transaction.prepare("INSERT INTO plain( filename, content ) VALUES( :fname, :content );") do |stmt|
93
+ stmt.execute( insert_data )
94
+ end
95
+
96
+ idx += 1
97
+ print "Processed #{idx} of #{lines.size}\r"
98
+ $stdout.flush
99
+ end
100
+ puts "Finalizing..."
101
+ end
102
+ puts "Took #{Time.now - before} seconds to insert #{idx} lines of #{lines.size}"
103
+ puts "Done Inserting"
104
+ end
105
+
106
+ doc_count = db.first_value_from( "SELECT count(*) from search" )
107
+
108
+ #
109
+ # Now lets do some searching for some various words
110
+ #
111
+
112
+ %w[ president salmon thanks ].each do |word|
113
+
114
+ count = 100
115
+ puts
116
+ puts "Searching for '#{word}' across #{doc_count} tweets"
117
+ puts "=" * 60
118
+
119
+ Benchmark.bm( 15 ) do |x|
120
+
121
+ #
122
+ # search using the fts search to get the cont of tweets with the given word
123
+ #
124
+ x.report('fts3: ') do
125
+ db.prepare( "SELECT count(filename) FROM search WHERE content MATCH '#{word}'" ) do |stmt|
126
+ count.times do
127
+ stmt.execute
128
+ end
129
+ end
130
+ end
131
+
132
+ #
133
+ # search using basic string matching in sqlite.
134
+ #
135
+ x.report('plain: ') do
136
+ db.prepare( "SELECT count(filename) FROM plain WHERE content LIKE '% #{word} %'" ) do |stmt|
137
+ count.times do
138
+ stmt.execute
139
+ end
140
+ end
141
+ end
142
+ end
143
+ end
144
+
@@ -17,10 +17,40 @@ void Init_amalgalite3_constants( )
17
17
  */
18
18
  VALUE mC = rb_define_module_under( mas, "Constants");
19
19
  /**
20
- * module encapsulating the SQLite3 C extension constants for Config
20
+ * module encapsulating the SQLite3 C extension constants for Config
21
21
  */
22
22
  VALUE mC_Config = rb_define_module_under(mC, "Config");
23
23
 
24
+ /**
25
+ * module encapsulating the SQLite3 C extension constants for DBStatus
26
+ */
27
+ VALUE mC_DBStatus = rb_define_module_under(mC, "DBStatus");
28
+
29
+ /**
30
+ * module encapsulating the SQLite3 C extension constants for DataType
31
+ */
32
+ VALUE mC_DataType = rb_define_module_under(mC, "DataType");
33
+
34
+ /**
35
+ * module encapsulating the SQLite3 C extension constants for Open
36
+ */
37
+ VALUE mC_Open = rb_define_module_under(mC, "Open");
38
+
39
+ /**
40
+ * module encapsulating the SQLite3 C extension constants for ResultCode
41
+ */
42
+ VALUE mC_ResultCode = rb_define_module_under(mC, "ResultCode");
43
+
44
+ /**
45
+ * module encapsulating the SQLite3 C extension constants for StatementStatus
46
+ */
47
+ VALUE mC_StatementStatus = rb_define_module_under(mC, "StatementStatus");
48
+
49
+ /**
50
+ * module encapsulating the SQLite3 C extension constants for Status
51
+ */
52
+ VALUE mC_Status = rb_define_module_under(mC, "Status");
53
+
24
54
  /* no meaningful autogenerated documentation -- constant is self explanatory ?*/
25
55
  rb_define_const(mC_Config, "GETMALLOC", INT2FIX(SQLITE_CONFIG_GETMALLOC));
26
56
 
@@ -66,11 +96,6 @@ void Init_amalgalite3_constants( )
66
96
  /* no meaningful autogenerated documentation -- constant is self explanatory ?*/
67
97
  rb_define_const(mC_Config, "SINGLETHREAD", INT2FIX(SQLITE_CONFIG_SINGLETHREAD));
68
98
 
69
- /**
70
- * module encapsulating the SQLite3 C extension constants for DBStatus
71
- */
72
- VALUE mC_DBStatus = rb_define_module_under(mC, "DBStatus");
73
-
74
99
  /* no meaningful autogenerated documentation -- constant is self explanatory ?*/
75
100
  rb_define_const(mC_DBStatus, "CACHE_USED", INT2FIX(SQLITE_DBSTATUS_CACHE_USED));
76
101
 
@@ -86,11 +111,6 @@ void Init_amalgalite3_constants( )
86
111
  /* no meaningful autogenerated documentation -- constant is self explanatory ?*/
87
112
  rb_define_const(mC_DBStatus, "STMT_USED", INT2FIX(SQLITE_DBSTATUS_STMT_USED));
88
113
 
89
- /**
90
- * module encapsulating the SQLite3 C extension constants for DataType
91
- */
92
- VALUE mC_DataType = rb_define_module_under(mC, "DataType");
93
-
94
114
  /* no meaningful autogenerated documentation -- constant is self explanatory ?*/
95
115
  rb_define_const(mC_DataType, "BLOB", INT2FIX(SQLITE_BLOB));
96
116
 
@@ -106,11 +126,6 @@ void Init_amalgalite3_constants( )
106
126
  /* no meaningful autogenerated documentation -- constant is self explanatory ?*/
107
127
  rb_define_const(mC_DataType, "TEXT", INT2FIX(SQLITE_TEXT));
108
128
 
109
- /**
110
- * module encapsulating the SQLite3 C extension constants for Open
111
- */
112
- VALUE mC_Open = rb_define_module_under(mC, "Open");
113
-
114
129
  /* no meaningful autogenerated documentation -- constant is self explanatory ?*/
115
130
  rb_define_const(mC_Open, "AUTOPROXY", INT2FIX(SQLITE_OPEN_AUTOPROXY));
116
131
 
@@ -165,11 +180,6 @@ void Init_amalgalite3_constants( )
165
180
  /* no meaningful autogenerated documentation -- constant is self explanatory ?*/
166
181
  rb_define_const(mC_Open, "WAL", INT2FIX(SQLITE_OPEN_WAL));
167
182
 
168
- /**
169
- * module encapsulating the SQLite3 C extension constants for ResultCode
170
- */
171
- VALUE mC_ResultCode = rb_define_module_under(mC, "ResultCode");
172
-
173
183
  /* 4 -- Callback routine requested an abort */
174
184
  rb_define_const(mC_ResultCode, "ABORT", INT2FIX(SQLITE_ABORT));
175
185
 
@@ -299,7 +309,7 @@ void Init_amalgalite3_constants( )
299
309
  /* 26 -- File opened that is not a database file */
300
310
  rb_define_const(mC_ResultCode, "NOTADB", INT2FIX(SQLITE_NOTADB));
301
311
 
302
- /* 12 -- NOT USED. Table or record not found */
312
+ /* 12 -- Unknown opcode in sqlite3_file_control() */
303
313
  rb_define_const(mC_ResultCode, "NOTFOUND", INT2FIX(SQLITE_NOTFOUND));
304
314
 
305
315
  /* no meaningful autogenerated documentation -- constant is self explanatory ?*/
@@ -326,11 +336,6 @@ void Init_amalgalite3_constants( )
326
336
  /* 18 -- String or BLOB exceeds size limit */
327
337
  rb_define_const(mC_ResultCode, "TOOBIG", INT2FIX(SQLITE_TOOBIG));
328
338
 
329
- /**
330
- * module encapsulating the SQLite3 C extension constants for StatementStatus
331
- */
332
- VALUE mC_StatementStatus = rb_define_module_under(mC, "StatementStatus");
333
-
334
339
  /* no meaningful autogenerated documentation -- constant is self explanatory ?*/
335
340
  rb_define_const(mC_StatementStatus, "AUTOINDEX", INT2FIX(SQLITE_STMTSTATUS_AUTOINDEX));
336
341
 
@@ -340,11 +345,6 @@ void Init_amalgalite3_constants( )
340
345
  /* no meaningful autogenerated documentation -- constant is self explanatory ?*/
341
346
  rb_define_const(mC_StatementStatus, "SORT", INT2FIX(SQLITE_STMTSTATUS_SORT));
342
347
 
343
- /**
344
- * module encapsulating the SQLite3 C extension constants for Status
345
- */
346
- VALUE mC_Status = rb_define_module_under(mC, "Status");
347
-
348
348
  /* no meaningful autogenerated documentation -- constant is self explanatory ?*/
349
349
  rb_define_const(mC_Status, "MALLOC_COUNT", INT2FIX(SQLITE_STATUS_MALLOC_COUNT));
350
350
 
@@ -8,6 +8,9 @@ $ruby = ARGV.shift if ARGV[0]
8
8
  # make available table and column meta data api
9
9
  $CFLAGS += " -DSQLITE_ENABLE_COLUMN_METADATA=1"
10
10
  $CFLAGS += " -DSQLITE_ENABLE_RTREE=1"
11
+ $CFLAGS += " -DSQLITE_ENABLE_FTS3=1"
12
+ $CFLAGS += " -DSQLITE_ENABLE_FTS3_PARENTHESIS=1"
13
+ $CFLAGS += " -DSQLITE_ENABLE_STAT2=1"
11
14
 
12
15
  # we compile sqlite the same way that the installation of ruby is compiled.
13
16
  if RbConfig::MAKEFILE_CONFIG['configure_args'].include?( "--enable-pthread" ) then
@@ -170,10 +170,14 @@ File.open(fname, "w+") do |f|
170
170
  CONSTANTS.keys.sort.each do |klass|
171
171
  const_set = CONSTANTS[klass]
172
172
  f.puts " /**"
173
- f.puts " * module encapsulating the SQLite3 C extension constants for #{klass} "
173
+ f.puts " * module encapsulating the SQLite3 C extension constants for #{klass}"
174
174
  f.puts " */"
175
175
  f.puts " VALUE mC_#{klass} = rb_define_module_under(mC, \"#{klass}\");"
176
176
  f.puts
177
+ end
178
+
179
+ CONSTANTS.keys.sort.each do |klass|
180
+ const_set = CONSTANTS[klass]
177
181
  const_set.keys.sort.each do |k|
178
182
  sql_const = const_set[k]
179
183
  const_doc = " /* no meaningful autogenerated documentation -- constant is self explanatory ?*/"
@@ -1,6 +1,6 @@
1
1
  /******************************************************************************
2
2
  ** This file is an amalgamation of many separate C source files from SQLite
3
- ** version 3.7.4. By combining all the individual C code files into this
3
+ ** version 3.7.5. By combining all the individual C code files into this
4
4
  ** single large file, the entire code can be compiled as a one translation
5
5
  ** unit. This allows many compilers to do optimizations that would not be
6
6
  ** possible if the files were compiled separately. Performance improvements
@@ -650,9 +650,9 @@ extern "C" {
650
650
  ** [sqlite3_libversion_number()], [sqlite3_sourceid()],
651
651
  ** [sqlite_version()] and [sqlite_source_id()].
652
652
  */
653
- #define SQLITE_VERSION "3.7.4"
654
- #define SQLITE_VERSION_NUMBER 3007004
655
- #define SQLITE_SOURCE_ID "2010-12-07 20:14:09 a586a4deeb25330037a49df295b36aaf624d0f45"
653
+ #define SQLITE_VERSION "3.7.5"
654
+ #define SQLITE_VERSION_NUMBER 3007005
655
+ #define SQLITE_SOURCE_ID "2011-01-28 17:03:50 ed759d5a9edb3bba5f48f243df47be29e3fe8cd7"
656
656
 
657
657
  /*
658
658
  ** CAPI3REF: Run-Time Library Version Numbers
@@ -933,7 +933,7 @@ SQLITE_API int sqlite3_exec(
933
933
  #define SQLITE_INTERRUPT 9 /* Operation terminated by sqlite3_interrupt()*/
934
934
  #define SQLITE_IOERR 10 /* Some kind of disk I/O error occurred */
935
935
  #define SQLITE_CORRUPT 11 /* The database disk image is malformed */
936
- #define SQLITE_NOTFOUND 12 /* NOT USED. Table or record not found */
936
+ #define SQLITE_NOTFOUND 12 /* Unknown opcode in sqlite3_file_control() */
937
937
  #define SQLITE_FULL 13 /* Insertion failed because database is full */
938
938
  #define SQLITE_CANTOPEN 14 /* Unable to open the database file */
939
939
  #define SQLITE_PROTOCOL 15 /* Database lock protocol error */
@@ -1165,7 +1165,9 @@ struct sqlite3_file {
1165
1165
  ** core reserves all opcodes less than 100 for its own use.
1166
1166
  ** A [SQLITE_FCNTL_LOCKSTATE | list of opcodes] less than 100 is available.
1167
1167
  ** Applications that define a custom xFileControl method should use opcodes
1168
- ** greater than 100 to avoid conflicts.
1168
+ ** greater than 100 to avoid conflicts. VFS implementations should
1169
+ ** return [SQLITE_NOTFOUND] for file control opcodes that they do not
1170
+ ** recognize.
1169
1171
  **
1170
1172
  ** The xSectorSize() method returns the sector size of the
1171
1173
  ** device that underlies the file. The sector size is the
@@ -1258,6 +1260,21 @@ struct sqlite3_io_methods {
1258
1260
  ** for the nominated database. Allocating database file space in large
1259
1261
  ** chunks (say 1MB at a time), may reduce file-system fragmentation and
1260
1262
  ** improve performance on some systems.
1263
+ **
1264
+ ** The [SQLITE_FCNTL_FILE_POINTER] opcode is used to obtain a pointer
1265
+ ** to the [sqlite3_file] object associated with a particular database
1266
+ ** connection. See the [sqlite3_file_control()] documentation for
1267
+ ** additional information.
1268
+ **
1269
+ ** ^(The [SQLITE_FCNTL_SYNC_OMITTED] opcode is generated internally by
1270
+ ** SQLite and sent to all VFSes in place of a call to the xSync method
1271
+ ** when the database connection has [PRAGMA synchronous] set to OFF.)^
1272
+ ** Some specialized VFSes need this signal in order to operate correctly
1273
+ ** when [PRAGMA synchronous | PRAGMA synchronous=OFF] is set, but most
1274
+ ** VFSes do not need this signal and should silently ignore this opcode.
1275
+ ** Applications should not call [sqlite3_file_control()] with this
1276
+ ** opcode as doing so may disrupt the operation of the specilized VFSes
1277
+ ** that do require it.
1261
1278
  */
1262
1279
  #define SQLITE_FCNTL_LOCKSTATE 1
1263
1280
  #define SQLITE_GET_LOCKPROXYFILE 2
@@ -1266,6 +1283,7 @@ struct sqlite3_io_methods {
1266
1283
  #define SQLITE_FCNTL_SIZE_HINT 5
1267
1284
  #define SQLITE_FCNTL_CHUNK_SIZE 6
1268
1285
  #define SQLITE_FCNTL_FILE_POINTER 7
1286
+ #define SQLITE_FCNTL_SYNC_OMITTED 8
1269
1287
 
1270
1288
 
1271
1289
  /*
@@ -2385,7 +2403,7 @@ SQLITE_API void sqlite3_free_table(char **result);
2385
2403
  ** NULL pointer if [sqlite3_malloc()] is unable to allocate enough
2386
2404
  ** memory to hold the resulting string.
2387
2405
  **
2388
- ** ^(In sqlite3_snprintf() routine is similar to "snprintf()" from
2406
+ ** ^(The sqlite3_snprintf() routine is similar to "snprintf()" from
2389
2407
  ** the standard C library. The result is written into the
2390
2408
  ** buffer supplied as the second parameter whose size is given by
2391
2409
  ** the first parameter. Note that the order of the
@@ -2404,6 +2422,8 @@ SQLITE_API void sqlite3_free_table(char **result);
2404
2422
  ** the zero terminator. So the longest string that can be completely
2405
2423
  ** written will be n-1 characters.
2406
2424
  **
2425
+ ** ^The sqlite3_vsnprintf() routine is a varargs version of sqlite3_snprintf().
2426
+ **
2407
2427
  ** These routines all implement some additional formatting
2408
2428
  ** options that are useful for constructing SQL statements.
2409
2429
  ** All of the usual printf() formatting options apply. In addition, there
@@ -2467,6 +2487,7 @@ SQLITE_API void sqlite3_free_table(char **result);
2467
2487
  SQLITE_API char *sqlite3_mprintf(const char*,...);
2468
2488
  SQLITE_API char *sqlite3_vmprintf(const char*, va_list);
2469
2489
  SQLITE_API char *sqlite3_snprintf(int,char*,const char*, ...);
2490
+ SQLITE_API char *sqlite3_vsnprintf(int,char*,const char*, va_list);
2470
2491
 
2471
2492
  /*
2472
2493
  ** CAPI3REF: Memory Allocation Subsystem
@@ -2844,7 +2865,7 @@ SQLITE_API void sqlite3_progress_handler(sqlite3*, int, int(*)(void*), void*);
2844
2865
  ** case the database must already exist, otherwise an error is returned.</dd>)^
2845
2866
  **
2846
2867
  ** ^(<dt>[SQLITE_OPEN_READWRITE] | [SQLITE_OPEN_CREATE]</dt>
2847
- ** <dd>The database is opened for reading and writing, and is creates it if
2868
+ ** <dd>The database is opened for reading and writing, and is created if
2848
2869
  ** it does not already exist. This is the behavior that is always used for
2849
2870
  ** sqlite3_open() and sqlite3_open16().</dd>)^
2850
2871
  ** </dl>
@@ -3193,14 +3214,31 @@ SQLITE_API const char *sqlite3_sql(sqlite3_stmt *pStmt);
3193
3214
  /*
3194
3215
  ** CAPI3REF: Determine If An SQL Statement Writes The Database
3195
3216
  **
3196
- ** ^The sqlite3_stmt_readonly(X) interface returns true (non-zero) if
3197
- ** the [prepared statement] X is [SELECT] statement and false (zero) if
3198
- ** X is an [INSERT], [UPDATE], [DELETE], CREATE, DROP, [ANALYZE],
3199
- ** [ALTER], or [REINDEX] statement.
3200
- ** If X is a NULL pointer or any other kind of statement, including but
3201
- ** not limited to [ATTACH], [DETACH], [COMMIT], [ROLLBACK], [RELEASE],
3202
- ** [SAVEPOINT], [PRAGMA], or [VACUUM] the result of sqlite3_stmt_readonly(X) is
3203
- ** undefined.
3217
+ ** ^The sqlite3_stmt_readonly(X) interface returns true (non-zero) if
3218
+ ** and only if the [prepared statement] X makes no direct changes to
3219
+ ** the content of the database file.
3220
+ **
3221
+ ** Note that [application-defined SQL functions] or
3222
+ ** [virtual tables] might change the database indirectly as a side effect.
3223
+ ** ^(For example, if an application defines a function "eval()" that
3224
+ ** calls [sqlite3_exec()], then the following SQL statement would
3225
+ ** change the database file through side-effects:
3226
+ **
3227
+ ** <blockquote><pre>
3228
+ ** SELECT eval('DELETE FROM t1') FROM t2;
3229
+ ** </pre></blockquote>
3230
+ **
3231
+ ** But because the [SELECT] statement does not change the database file
3232
+ ** directly, sqlite3_stmt_readonly() would still return true.)^
3233
+ **
3234
+ ** ^Transaction control statements such as [BEGIN], [COMMIT], [ROLLBACK],
3235
+ ** [SAVEPOINT], and [RELEASE] cause sqlite3_stmt_readonly() to return true,
3236
+ ** since the statements themselves do not actually modify the database but
3237
+ ** rather they control the timing of when other statements modify the
3238
+ ** database. ^The [ATTACH] and [DETACH] statements also cause
3239
+ ** sqlite3_stmt_readonly() to return true since, while those statements
3240
+ ** change the configuration of a database connection, they do not make
3241
+ ** changes to the content of the database files on disk.
3204
3242
  */
3205
3243
  SQLITE_API int sqlite3_stmt_readonly(sqlite3_stmt *pStmt);
3206
3244
 
@@ -3594,13 +3632,17 @@ SQLITE_API const void *sqlite3_column_decltype16(sqlite3_stmt*,int);
3594
3632
  ** be the case that the same database connection is being used by two or
3595
3633
  ** more threads at the same moment in time.
3596
3634
  **
3597
- ** For all versions of SQLite up to and including 3.6.23.1, it was required
3598
- ** after sqlite3_step() returned anything other than [SQLITE_ROW] that
3599
- ** [sqlite3_reset()] be called before any subsequent invocation of
3600
- ** sqlite3_step(). Failure to invoke [sqlite3_reset()] in this way would
3601
- ** result in an [SQLITE_MISUSE] return from sqlite3_step(). But after
3602
- ** version 3.6.23.1, sqlite3_step() began calling [sqlite3_reset()]
3603
- ** automatically in this circumstance rather than returning [SQLITE_MISUSE].
3635
+ ** For all versions of SQLite up to and including 3.6.23.1, a call to
3636
+ ** [sqlite3_reset()] was required after sqlite3_step() returned anything
3637
+ ** other than [SQLITE_ROW] before any subsequent invocation of
3638
+ ** sqlite3_step(). Failure to reset the prepared statement using
3639
+ ** [sqlite3_reset()] would result in an [SQLITE_MISUSE] return from
3640
+ ** sqlite3_step(). But after version 3.6.23.1, sqlite3_step() began
3641
+ ** calling [sqlite3_reset()] automatically in this circumstance rather
3642
+ ** than returning [SQLITE_MISUSE]. This is not considered a compatibility
3643
+ ** break because any application that ever receives an SQLITE_MISUSE error
3644
+ ** is broken by definition. The [SQLITE_OMIT_AUTORESET] compile-time option
3645
+ ** can be used to restore the legacy behavior.
3604
3646
  **
3605
3647
  ** <b>Goofy Interface Alert:</b> In the legacy interface, the sqlite3_step()
3606
3648
  ** API always returns a generic error code, [SQLITE_ERROR], following any
@@ -3937,7 +3979,7 @@ SQLITE_API int sqlite3_reset(sqlite3_stmt *pStmt);
3937
3979
  ** ^(The fifth parameter is an arbitrary pointer. The implementation of the
3938
3980
  ** function can gain access to this pointer using [sqlite3_user_data()].)^
3939
3981
  **
3940
- ** ^The seventh, eighth and ninth parameters, xFunc, xStep and xFinal, are
3982
+ ** ^The sixth, seventh and eighth parameters, xFunc, xStep and xFinal, are
3941
3983
  ** pointers to C-language functions that implement the SQL function or
3942
3984
  ** aggregate. ^A scalar SQL function requires an implementation of the xFunc
3943
3985
  ** callback only; NULL pointers must be passed as the xStep and xFinal
@@ -3946,7 +3988,7 @@ SQLITE_API int sqlite3_reset(sqlite3_stmt *pStmt);
3946
3988
  ** SQL function or aggregate, pass NULL poiners for all three function
3947
3989
  ** callbacks.
3948
3990
  **
3949
- ** ^(If the tenth parameter to sqlite3_create_function_v2() is not NULL,
3991
+ ** ^(If the ninth parameter to sqlite3_create_function_v2() is not NULL,
3950
3992
  ** then it is destructor for the application data pointer.
3951
3993
  ** The destructor is invoked when the function is deleted, either by being
3952
3994
  ** overloaded or when the database connection closes.)^
@@ -4050,7 +4092,7 @@ SQLITE_API SQLITE_DEPRECATED int sqlite3_memory_alarm(void(*)(void*,sqlite3_int6
4050
4092
  ** The xFunc (for scalar functions) or xStep (for aggregates) parameters
4051
4093
  ** to [sqlite3_create_function()] and [sqlite3_create_function16()]
4052
4094
  ** define callbacks that implement the SQL functions and aggregates.
4053
- ** The 4th parameter to these callbacks is an array of pointers to
4095
+ ** The 3rd parameter to these callbacks is an array of pointers to
4054
4096
  ** [protected sqlite3_value] objects. There is one [sqlite3_value] object for
4055
4097
  ** each parameter to the SQL function. These routines are used to
4056
4098
  ** extract values from the [sqlite3_value] objects.
@@ -5778,7 +5820,8 @@ SQLITE_API int sqlite3_mutex_notheld(sqlite3_mutex*);
5778
5820
  #define SQLITE_MUTEX_STATIC_OPEN 4 /* sqlite3BtreeOpen() */
5779
5821
  #define SQLITE_MUTEX_STATIC_PRNG 5 /* sqlite3_random() */
5780
5822
  #define SQLITE_MUTEX_STATIC_LRU 6 /* lru page list */
5781
- #define SQLITE_MUTEX_STATIC_LRU2 7 /* lru page list */
5823
+ #define SQLITE_MUTEX_STATIC_LRU2 7 /* NOT USED */
5824
+ #define SQLITE_MUTEX_STATIC_PMEM 7 /* sqlite3PageMalloc() */
5782
5825
 
5783
5826
  /*
5784
5827
  ** CAPI3REF: Retrieve the mutex for a database connection
@@ -5929,7 +5972,8 @@ SQLITE_API int sqlite3_status(int op, int *pCurrent, int *pHighwater, int resetF
5929
5972
  ** The value written into the *pCurrent parameter is undefined.</dd>)^
5930
5973
  **
5931
5974
  ** ^(<dt>SQLITE_STATUS_MALLOC_COUNT</dt>
5932
- ** <dd>This parameter records the number of separate memory allocations.</dd>)^
5975
+ ** <dd>This parameter records the number of separate memory allocations
5976
+ ** currently checked out.</dd>)^
5933
5977
  **
5934
5978
  ** ^(<dt>SQLITE_STATUS_PAGECACHE_USED</dt>
5935
5979
  ** <dd>This parameter returns the number of pages used out of the
@@ -6035,6 +6079,28 @@ SQLITE_API int sqlite3_db_status(sqlite3*, int op, int *pCur, int *pHiwtr, int r
6035
6079
  ** <dd>This parameter returns the number of lookaside memory slots currently
6036
6080
  ** checked out.</dd>)^
6037
6081
  **
6082
+ ** ^(<dt>SQLITE_DBSTATUS_LOOKASIDE_HIT</dt>
6083
+ ** <dd>This parameter returns the number malloc attempts that were
6084
+ ** satisfied using lookaside memory. Only the high-water value is meaningful;
6085
+ ** the current value is always zero.
6086
+ ** checked out.</dd>)^
6087
+ **
6088
+ ** ^(<dt>SQLITE_DBSTATUS_LOOKASIDE_MISS_SIZE</dt>
6089
+ ** <dd>This parameter returns the number malloc attempts that might have
6090
+ ** been satisfied using lookaside memory but failed due to the amount of
6091
+ ** memory requested being larger than the lookaside slot size.
6092
+ ** Only the high-water value is meaningful;
6093
+ ** the current value is always zero.
6094
+ ** checked out.</dd>)^
6095
+ **
6096
+ ** ^(<dt>SQLITE_DBSTATUS_LOOKASIDE_MISS_FULL</dt>
6097
+ ** <dd>This parameter returns the number malloc attempts that might have
6098
+ ** been satisfied using lookaside memory but failed due to all lookaside
6099
+ ** memory already being in use.
6100
+ ** Only the high-water value is meaningful;
6101
+ ** the current value is always zero.
6102
+ ** checked out.</dd>)^
6103
+ **
6038
6104
  ** ^(<dt>SQLITE_DBSTATUS_CACHE_USED</dt>
6039
6105
  ** <dd>This parameter returns the approximate number of of bytes of heap
6040
6106
  ** memory used by all pager caches associated with the database connection.)^
@@ -6057,11 +6123,14 @@ SQLITE_API int sqlite3_db_status(sqlite3*, int op, int *pCur, int *pHiwtr, int r
6057
6123
  ** </dd>
6058
6124
  ** </dl>
6059
6125
  */
6060
- #define SQLITE_DBSTATUS_LOOKASIDE_USED 0
6061
- #define SQLITE_DBSTATUS_CACHE_USED 1
6062
- #define SQLITE_DBSTATUS_SCHEMA_USED 2
6063
- #define SQLITE_DBSTATUS_STMT_USED 3
6064
- #define SQLITE_DBSTATUS_MAX 3 /* Largest defined DBSTATUS */
6126
+ #define SQLITE_DBSTATUS_LOOKASIDE_USED 0
6127
+ #define SQLITE_DBSTATUS_CACHE_USED 1
6128
+ #define SQLITE_DBSTATUS_SCHEMA_USED 2
6129
+ #define SQLITE_DBSTATUS_STMT_USED 3
6130
+ #define SQLITE_DBSTATUS_LOOKASIDE_HIT 4
6131
+ #define SQLITE_DBSTATUS_LOOKASIDE_MISS_SIZE 5
6132
+ #define SQLITE_DBSTATUS_LOOKASIDE_MISS_FULL 6
6133
+ #define SQLITE_DBSTATUS_MAX 6 /* Largest defined DBSTATUS */
6065
6134
 
6066
6135
 
6067
6136
  /*
@@ -6189,11 +6258,13 @@ typedef struct sqlite3_pcache sqlite3_pcache;
6189
6258
  ** first parameter, szPage, is the size in bytes of the pages that must
6190
6259
  ** be allocated by the cache. ^szPage will not be a power of two. ^szPage
6191
6260
  ** will the page size of the database file that is to be cached plus an
6192
- ** increment (here called "R") of about 100 or 200. SQLite will use the
6261
+ ** increment (here called "R") of less than 250. SQLite will use the
6193
6262
  ** extra R bytes on each page to store metadata about the underlying
6194
6263
  ** database page on disk. The value of R depends
6195
6264
  ** on the SQLite version, the target platform, and how SQLite was compiled.
6196
- ** ^R is constant for a particular build of SQLite. ^The second argument to
6265
+ ** ^(R is constant for a particular build of SQLite. Except, there are two
6266
+ ** distinct values of R when SQLite is compiled with the proprietary
6267
+ ** ZIPVFS extension.)^ ^The second argument to
6197
6268
  ** xCreate(), bPurgeable, is true if the cache being created will
6198
6269
  ** be used to cache database pages of a file stored on disk, or
6199
6270
  ** false if it is used for an in-memory database. The cache implementation
@@ -6225,7 +6296,7 @@ typedef struct sqlite3_pcache sqlite3_pcache;
6225
6296
  ** If the requested page is already in the page cache, then the page cache
6226
6297
  ** implementation must return a pointer to the page buffer with its content
6227
6298
  ** intact. If the requested page is not already in the cache, then the
6228
- ** behavior of the cache implementation should use the value of the createFlag
6299
+ ** cache implementation should use the value of the createFlag
6229
6300
  ** parameter to help it determined what action to take:
6230
6301
  **
6231
6302
  ** <table border=1 width=85% align=center>
@@ -6309,11 +6380,12 @@ typedef struct sqlite3_backup sqlite3_backup;
6309
6380
  **
6310
6381
  ** See Also: [Using the SQLite Online Backup API]
6311
6382
  **
6312
- ** ^Exclusive access is required to the destination database for the
6313
- ** duration of the operation. ^However the source database is only
6314
- ** read-locked while it is actually being read; it is not locked
6315
- ** continuously for the entire backup operation. ^Thus, the backup may be
6316
- ** performed on a live source database without preventing other users from
6383
+ ** ^SQLite holds a write transaction open on the destination database file
6384
+ ** for the duration of the backup operation.
6385
+ ** ^The source database is read-locked only while it is being read;
6386
+ ** it is not locked continuously for the entire backup operation.
6387
+ ** ^Thus, the backup may be performed on a live source database without
6388
+ ** preventing other database connections from
6317
6389
  ** reading or writing to the source database while the backup is underway.
6318
6390
  **
6319
6391
  ** ^(To perform a backup operation:
@@ -6340,11 +6412,11 @@ typedef struct sqlite3_backup sqlite3_backup;
6340
6412
  ** sqlite3_backup_init(D,N,S,M) identify the [database connection]
6341
6413
  ** and database name of the source database, respectively.
6342
6414
  ** ^The source and destination [database connections] (parameters S and D)
6343
- ** must be different or else sqlite3_backup_init(D,N,S,M) will file with
6415
+ ** must be different or else sqlite3_backup_init(D,N,S,M) will fail with
6344
6416
  ** an error.
6345
6417
  **
6346
6418
  ** ^If an error occurs within sqlite3_backup_init(D,N,S,M), then NULL is
6347
- ** returned and an error code and error message are store3d in the
6419
+ ** returned and an error code and error message are stored in the
6348
6420
  ** destination [database connection] D.
6349
6421
  ** ^The error code and message for the failed call to sqlite3_backup_init()
6350
6422
  ** can be retrieved using the [sqlite3_errcode()], [sqlite3_errmsg()], and/or
@@ -6361,7 +6433,7 @@ typedef struct sqlite3_backup sqlite3_backup;
6361
6433
  ** the source and destination databases specified by [sqlite3_backup] object B.
6362
6434
  ** ^If N is negative, all remaining source pages are copied.
6363
6435
  ** ^If sqlite3_backup_step(B,N) successfully copies N pages and there
6364
- ** are still more pages to be copied, then the function resturns [SQLITE_OK].
6436
+ ** are still more pages to be copied, then the function returns [SQLITE_OK].
6365
6437
  ** ^If sqlite3_backup_step(B,N) successfully finishes copying all pages
6366
6438
  ** from source to destination, then it returns [SQLITE_DONE].
6367
6439
  ** ^If an error occurs while running sqlite3_backup_step(B,N),
@@ -6375,7 +6447,7 @@ typedef struct sqlite3_backup sqlite3_backup;
6375
6447
  ** <li> the destination database was opened read-only, or
6376
6448
  ** <li> the destination database is using write-ahead-log journaling
6377
6449
  ** and the destination and source page sizes differ, or
6378
- ** <li> The destination database is an in-memory database and the
6450
+ ** <li> the destination database is an in-memory database and the
6379
6451
  ** destination and source page sizes differ.
6380
6452
  ** </ol>)^
6381
6453
  **
@@ -6706,7 +6778,8 @@ SQLITE_API void *sqlite3_wal_hook(
6706
6778
  ** from SQL.
6707
6779
  **
6708
6780
  ** ^Every new [database connection] defaults to having the auto-checkpoint
6709
- ** enabled with a threshold of 1000 pages. The use of this interface
6781
+ ** enabled with a threshold of 1000 or [SQLITE_DEFAULT_WAL_AUTOCHECKPOINT]
6782
+ ** pages. The use of this interface
6710
6783
  ** is only necessary if the default setting is found to be suboptimal
6711
6784
  ** for a particular application.
6712
6785
  */
@@ -8877,6 +8950,7 @@ struct Lookaside {
8877
8950
  u8 bMalloced; /* True if pStart obtained from sqlite3_malloc() */
8878
8951
  int nOut; /* Number of buffers currently checked out */
8879
8952
  int mxOut; /* Highwater mark for nOut */
8953
+ int anStat[3]; /* 0: hits. 1: size misses. 2: full misses */
8880
8954
  LookasideSlot *pFree; /* List of available buffers */
8881
8955
  void *pStart; /* First byte of available memory space */
8882
8956
  void *pEnd; /* First byte past end of available space */
@@ -8955,6 +9029,7 @@ struct sqlite3 {
8955
9029
  struct Vdbe *pVdbe; /* List of active virtual machines */
8956
9030
  int activeVdbeCnt; /* Number of VDBEs currently executing */
8957
9031
  int writeVdbeCnt; /* Number of active VDBEs that are writing */
9032
+ int vdbeExecCnt; /* Number of nested calls to VdbeExec() */
8958
9033
  void (*xTrace)(void*,const char*); /* Trace function */
8959
9034
  void *pTraceArg; /* Argument to the trace function */
8960
9035
  void (*xProfile)(void*,const char*,u64); /* Profiling function */
@@ -11727,6 +11802,9 @@ static const char * const azCompileOpt[] = {
11727
11802
  #ifdef SQLITE_OMIT_AUTOMATIC_INDEX
11728
11803
  "OMIT_AUTOMATIC_INDEX",
11729
11804
  #endif
11805
+ #ifdef SQLITE_OMIT_AUTORESET
11806
+ "OMIT_AUTORESET",
11807
+ #endif
11730
11808
  #ifdef SQLITE_OMIT_AUTOVACUUM
11731
11809
  "OMIT_AUTOVACUUM",
11732
11810
  #endif
@@ -12002,16 +12080,14 @@ typedef unsigned char Bool;
12002
12080
  **
12003
12081
  ** Every cursor that the virtual machine has open is represented by an
12004
12082
  ** instance of the following structure.
12005
- **
12006
- ** If the VdbeCursor.isTriggerRow flag is set it means that this cursor is
12007
- ** really a single row that represents the NEW or OLD pseudo-table of
12008
- ** a row trigger. The data for the row is stored in VdbeCursor.pData and
12009
- ** the rowid is in VdbeCursor.iKey.
12010
12083
  */
12011
12084
  struct VdbeCursor {
12012
12085
  BtCursor *pCursor; /* The cursor structure of the backend */
12086
+ Btree *pBt; /* Separate file holding temporary table */
12087
+ KeyInfo *pKeyInfo; /* Info about index keys needed by index cursors */
12013
12088
  int iDb; /* Index of cursor database in db->aDb[] (or -1) */
12014
- i64 lastRowid; /* Last rowid from a Next or NextIdx operation */
12089
+ int pseudoTableReg; /* Register holding pseudotable content. */
12090
+ int nField; /* Number of fields in the header */
12015
12091
  Bool zeroed; /* True if zeroed out and ready for reuse */
12016
12092
  Bool rowidIsValid; /* True if lastRowid is valid */
12017
12093
  Bool atFirst; /* True if pointing to first entry */
@@ -12021,14 +12097,11 @@ struct VdbeCursor {
12021
12097
  Bool isTable; /* True if a table requiring integer keys */
12022
12098
  Bool isIndex; /* True if an index containing keys only - no data */
12023
12099
  Bool isOrdered; /* True if the underlying table is BTREE_UNORDERED */
12024
- i64 movetoTarget; /* Argument to the deferred sqlite3BtreeMoveto() */
12025
- Btree *pBt; /* Separate file holding temporary table */
12026
- int pseudoTableReg; /* Register holding pseudotable content. */
12027
- KeyInfo *pKeyInfo; /* Info about index keys needed by index cursors */
12028
- int nField; /* Number of fields in the header */
12029
- i64 seqCount; /* Sequence counter */
12030
12100
  sqlite3_vtab_cursor *pVtabCursor; /* The cursor for a virtual table */
12031
12101
  const sqlite3_module *pModule; /* Module for cursor pVtabCursor */
12102
+ i64 seqCount; /* Sequence counter */
12103
+ i64 movetoTarget; /* Argument to the deferred sqlite3BtreeMoveto() */
12104
+ i64 lastRowid; /* Last rowid from a Next or NextIdx operation */
12032
12105
 
12033
12106
  /* Result of last sqlite3BtreeMoveto() done by an OP_NotExists or
12034
12107
  ** OP_IsUnique opcode on this cursor. */
@@ -12100,25 +12173,19 @@ struct VdbeFrame {
12100
12173
  /*
12101
12174
  ** Internally, the vdbe manipulates nearly all SQL values as Mem
12102
12175
  ** structures. Each Mem struct may cache multiple representations (string,
12103
- ** integer etc.) of the same value. A value (and therefore Mem structure)
12104
- ** has the following properties:
12105
- **
12106
- ** Each value has a manifest type. The manifest type of the value stored
12107
- ** in a Mem struct is returned by the MemType(Mem*) macro. The type is
12108
- ** one of SQLITE_NULL, SQLITE_INTEGER, SQLITE_REAL, SQLITE_TEXT or
12109
- ** SQLITE_BLOB.
12176
+ ** integer etc.) of the same value.
12110
12177
  */
12111
12178
  struct Mem {
12179
+ sqlite3 *db; /* The associated database connection */
12180
+ char *z; /* String or BLOB value */
12181
+ double r; /* Real value */
12112
12182
  union {
12113
- i64 i; /* Integer value. */
12183
+ i64 i; /* Integer value used when MEM_Int is set in flags */
12114
12184
  int nZero; /* Used when bit MEM_Zero is set in flags */
12115
12185
  FuncDef *pDef; /* Used only when flags==MEM_Agg */
12116
12186
  RowSet *pRowSet; /* Used only when flags==MEM_RowSet */
12117
12187
  VdbeFrame *pFrame; /* Used when flags==MEM_Frame */
12118
12188
  } u;
12119
- double r; /* Real value */
12120
- sqlite3 *db; /* The associated database connection */
12121
- char *z; /* String or BLOB value */
12122
12189
  int n; /* Number of characters in string value, excluding '\0' */
12123
12190
  u16 flags; /* Some combination of MEM_Null, MEM_Str, MEM_Dyn, etc. */
12124
12191
  u8 type; /* One of SQLITE_NULL, SQLITE_TEXT, SQLITE_INTEGER, etc */
@@ -12142,9 +12209,6 @@ struct Mem {
12142
12209
  ** database (see below for exceptions). If the MEM_Term flag is also
12143
12210
  ** set, then the string is nul terminated. The MEM_Int and MEM_Real
12144
12211
  ** flags may coexist with the MEM_Str flag.
12145
- **
12146
- ** Multiple of these values can appear in Mem.flags. But only one
12147
- ** at a time can appear in Mem.type.
12148
12212
  */
12149
12213
  #define MEM_Null 0x0001 /* Value is NULL */
12150
12214
  #define MEM_Str 0x0002 /* Value is a string */
@@ -12227,23 +12291,11 @@ struct sqlite3_context {
12227
12291
  CollSeq *pColl; /* Collating sequence */
12228
12292
  };
12229
12293
 
12230
- /*
12231
- ** A Set structure is used for quick testing to see if a value
12232
- ** is part of a small set. Sets are used to implement code like
12233
- ** this:
12234
- ** x.y IN ('hi','hoo','hum')
12235
- */
12236
- typedef struct Set Set;
12237
- struct Set {
12238
- Hash hash; /* A set is just a hash table */
12239
- HashElem *prev; /* Previously accessed hash elemen */
12240
- };
12241
-
12242
12294
  /*
12243
12295
  ** An instance of the virtual machine. This structure contains the complete
12244
12296
  ** state of the virtual machine.
12245
12297
  **
12246
- ** The "sqlite3_stmt" structure pointer that is returned by sqlite3_compile()
12298
+ ** The "sqlite3_stmt" structure pointer that is returned by sqlite3_prepare()
12247
12299
  ** is really a pointer to an instance of this structure.
12248
12300
  **
12249
12301
  ** The Vdbe.inVtabMethod variable is set to non-zero for the duration of
@@ -12256,31 +12308,31 @@ struct Set {
12256
12308
  */
12257
12309
  struct Vdbe {
12258
12310
  sqlite3 *db; /* The database connection that owns this statement */
12259
- Vdbe *pPrev,*pNext; /* Linked list of VDBEs with the same Vdbe.db */
12311
+ Op *aOp; /* Space to hold the virtual machine's program */
12312
+ Mem *aMem; /* The memory locations */
12313
+ Mem **apArg; /* Arguments to currently executing user function */
12314
+ Mem *aColName; /* Column names to return */
12315
+ Mem *pResultSet; /* Pointer to an array of results */
12316
+ int nMem; /* Number of memory locations currently allocated */
12260
12317
  int nOp; /* Number of instructions in the program */
12261
12318
  int nOpAlloc; /* Number of slots allocated for aOp[] */
12262
- Op *aOp; /* Space to hold the virtual machine's program */
12263
12319
  int nLabel; /* Number of labels used */
12264
12320
  int nLabelAlloc; /* Number of slots allocated in aLabel[] */
12265
12321
  int *aLabel; /* Space to hold the labels */
12266
- Mem **apArg; /* Arguments to currently executing user function */
12267
- Mem *aColName; /* Column names to return */
12268
- Mem *pResultSet; /* Pointer to an array of results */
12269
12322
  u16 nResColumn; /* Number of columns in one row of the result set */
12270
12323
  u16 nCursor; /* Number of slots in apCsr[] */
12324
+ u32 magic; /* Magic number for sanity checking */
12325
+ char *zErrMsg; /* Error message written here */
12326
+ Vdbe *pPrev,*pNext; /* Linked list of VDBEs with the same Vdbe.db */
12271
12327
  VdbeCursor **apCsr; /* One element of this array for each open cursor */
12272
- u8 errorAction; /* Recovery action to do in case of an error */
12273
- u8 okVar; /* True if azVar[] has been initialized */
12274
- ynVar nVar; /* Number of entries in aVar[] */
12275
12328
  Mem *aVar; /* Values for the OP_Variable opcode. */
12276
12329
  char **azVar; /* Name of variables */
12277
- u32 magic; /* Magic number for sanity checking */
12278
- int nMem; /* Number of memory locations currently allocated */
12279
- Mem *aMem; /* The memory locations */
12330
+ ynVar nVar; /* Number of entries in aVar[] */
12280
12331
  u32 cacheCtr; /* VdbeCursor row cache generation counter */
12281
12332
  int pc; /* The program counter */
12282
12333
  int rc; /* Value to return */
12283
- char *zErrMsg; /* Error message written here */
12334
+ u8 errorAction; /* Recovery action to do in case of an error */
12335
+ u8 okVar; /* True if azVar[] has been initialized */
12284
12336
  u8 explain; /* True if EXPLAIN present on SQL command */
12285
12337
  u8 changeCntOn; /* True to update the change-counter */
12286
12338
  u8 expired; /* True if the VM needs to be recompiled */
@@ -12292,14 +12344,16 @@ struct Vdbe {
12292
12344
  u8 isPrepareV2; /* True if prepared with prepare_v2() */
12293
12345
  int nChange; /* Number of db changes made since last reset */
12294
12346
  int btreeMask; /* Bitmask of db->aDb[] entries referenced */
12295
- i64 startTime; /* Time when query started - used for profiling */
12296
- BtreeMutexArray aMutex; /* An array of Btree used here and needing locks */
12347
+ int iStatement; /* Statement number (or 0 if has not opened stmt) */
12297
12348
  int aCounter[3]; /* Counters used by sqlite3_stmt_status() */
12298
- char *zSql; /* Text of the SQL statement that generated this */
12299
- void *pFree; /* Free this when deleting the vdbe */
12349
+ BtreeMutexArray aMutex; /* An array of Btree used here and needing locks */
12350
+ #ifndef SQLITE_OMIT_TRACE
12351
+ i64 startTime; /* Time when query started - used for profiling */
12352
+ #endif
12300
12353
  i64 nFkConstraint; /* Number of imm. FK constraints this VM */
12301
12354
  i64 nStmtDefCons; /* Number of def. constraints when stmt started */
12302
- int iStatement; /* Statement number (or 0 if has not opened stmt) */
12355
+ char *zSql; /* Text of the SQL statement that generated this */
12356
+ void *pFree; /* Free this when deleting the vdbe */
12303
12357
  #ifdef SQLITE_DEBUG
12304
12358
  FILE *trace; /* Write an execution trace here, if not NULL */
12305
12359
  #endif
@@ -12509,6 +12563,22 @@ SQLITE_API int sqlite3_db_status(
12509
12563
  break;
12510
12564
  }
12511
12565
 
12566
+ case SQLITE_DBSTATUS_LOOKASIDE_HIT:
12567
+ case SQLITE_DBSTATUS_LOOKASIDE_MISS_SIZE:
12568
+ case SQLITE_DBSTATUS_LOOKASIDE_MISS_FULL: {
12569
+ testcase( op==SQLITE_DBSTATUS_LOOKASIDE_HIT );
12570
+ testcase( op==SQLITE_DBSTATUS_LOOKASIDE_MISS_SIZE );
12571
+ testcase( op==SQLITE_DBSTATUS_LOOKASIDE_MISS_FULL );
12572
+ assert( (op-SQLITE_DBSTATUS_LOOKASIDE_HIT)>=0 );
12573
+ assert( (op-SQLITE_DBSTATUS_LOOKASIDE_HIT)<3 );
12574
+ *pCurrent = 0;
12575
+ *pHighwater = db->lookaside.anStat[op - SQLITE_DBSTATUS_LOOKASIDE_HIT];
12576
+ if( resetFlag ){
12577
+ db->lookaside.anStat[op - SQLITE_DBSTATUS_LOOKASIDE_HIT] = 0;
12578
+ }
12579
+ break;
12580
+ }
12581
+
12512
12582
  /*
12513
12583
  ** Return an approximation for the amount of memory currently used
12514
12584
  ** by all pagers associated with the given database connection. The
@@ -16867,7 +16937,7 @@ static int pthreadMutexEnd(void){ return SQLITE_OK; }
16867
16937
  ** <li> SQLITE_MUTEX_STATIC_MEM2
16868
16938
  ** <li> SQLITE_MUTEX_STATIC_PRNG
16869
16939
  ** <li> SQLITE_MUTEX_STATIC_LRU
16870
- ** <li> SQLITE_MUTEX_STATIC_LRU2
16940
+ ** <li> SQLITE_MUTEX_STATIC_PMEM
16871
16941
  ** </ul>
16872
16942
  **
16873
16943
  ** The first two constants cause sqlite3_mutex_alloc() to create
@@ -17277,7 +17347,7 @@ static int winMutexEnd(void){
17277
17347
  ** <li> SQLITE_MUTEX_STATIC_MEM2
17278
17348
  ** <li> SQLITE_MUTEX_STATIC_PRNG
17279
17349
  ** <li> SQLITE_MUTEX_STATIC_LRU
17280
- ** <li> SQLITE_MUTEX_STATIC_LRU2
17350
+ ** <li> SQLITE_MUTEX_STATIC_PMEM
17281
17351
  ** </ul>
17282
17352
  **
17283
17353
  ** The first two constants cause sqlite3_mutex_alloc() to create
@@ -18070,14 +18140,20 @@ SQLITE_PRIVATE void *sqlite3DbMallocRaw(sqlite3 *db, int n){
18070
18140
  if( db->mallocFailed ){
18071
18141
  return 0;
18072
18142
  }
18073
- if( db->lookaside.bEnabled && n<=db->lookaside.sz
18074
- && (pBuf = db->lookaside.pFree)!=0 ){
18075
- db->lookaside.pFree = pBuf->pNext;
18076
- db->lookaside.nOut++;
18077
- if( db->lookaside.nOut>db->lookaside.mxOut ){
18078
- db->lookaside.mxOut = db->lookaside.nOut;
18143
+ if( db->lookaside.bEnabled ){
18144
+ if( n>db->lookaside.sz ){
18145
+ db->lookaside.anStat[1]++;
18146
+ }else if( (pBuf = db->lookaside.pFree)==0 ){
18147
+ db->lookaside.anStat[2]++;
18148
+ }else{
18149
+ db->lookaside.pFree = pBuf->pNext;
18150
+ db->lookaside.nOut++;
18151
+ db->lookaside.anStat[0]++;
18152
+ if( db->lookaside.nOut>db->lookaside.mxOut ){
18153
+ db->lookaside.mxOut = db->lookaside.nOut;
18154
+ }
18155
+ return (void*)pBuf;
18079
18156
  }
18080
- return (void*)pBuf;
18081
18157
  }
18082
18158
  }
18083
18159
  #else
@@ -18989,6 +19065,7 @@ SQLITE_PRIVATE void sqlite3StrAccumAppend(StrAccum *p, const char *z, int N){
18989
19065
  return;
18990
19066
  }
18991
19067
  }else{
19068
+ char *zOld = (p->zText==p->zBase ? 0 : p->zText);
18992
19069
  i64 szNew = p->nChar;
18993
19070
  szNew += N + 1;
18994
19071
  if( szNew > p->mxAlloc ){
@@ -18999,13 +19076,12 @@ SQLITE_PRIVATE void sqlite3StrAccumAppend(StrAccum *p, const char *z, int N){
18999
19076
  p->nAlloc = (int)szNew;
19000
19077
  }
19001
19078
  if( p->useMalloc==1 ){
19002
- zNew = sqlite3DbMallocRaw(p->db, p->nAlloc );
19079
+ zNew = sqlite3DbRealloc(p->db, zOld, p->nAlloc);
19003
19080
  }else{
19004
- zNew = sqlite3_malloc(p->nAlloc);
19081
+ zNew = sqlite3_realloc(zOld, p->nAlloc);
19005
19082
  }
19006
19083
  if( zNew ){
19007
- memcpy(zNew, p->zText, p->nChar);
19008
- sqlite3StrAccumReset(p);
19084
+ if( zOld==0 ) memcpy(zNew, p->zText, p->nChar);
19009
19085
  p->zText = zNew;
19010
19086
  }else{
19011
19087
  p->mallocFailed = 1;
@@ -19160,21 +19236,28 @@ SQLITE_API char *sqlite3_mprintf(const char *zFormat, ...){
19160
19236
  ** current locale settings. This is important for SQLite because we
19161
19237
  ** are not able to use a "," as the decimal point in place of "." as
19162
19238
  ** specified by some locales.
19239
+ **
19240
+ ** Oops: The first two arguments of sqlite3_snprintf() are backwards
19241
+ ** from the snprintf() standard. Unfortunately, it is too late to change
19242
+ ** this without breaking compatibility, so we just have to live with the
19243
+ ** mistake.
19244
+ **
19245
+ ** sqlite3_vsnprintf() is the varargs version.
19163
19246
  */
19164
- SQLITE_API char *sqlite3_snprintf(int n, char *zBuf, const char *zFormat, ...){
19165
- char *z;
19166
- va_list ap;
19247
+ SQLITE_API char *sqlite3_vsnprintf(int n, char *zBuf, const char *zFormat, va_list ap){
19167
19248
  StrAccum acc;
19168
-
19169
- if( n<=0 ){
19170
- return zBuf;
19171
- }
19249
+ if( n<=0 ) return zBuf;
19172
19250
  sqlite3StrAccumInit(&acc, zBuf, n, 0);
19173
19251
  acc.useMalloc = 0;
19174
- va_start(ap,zFormat);
19175
19252
  sqlite3VXPrintf(&acc, 0, zFormat, ap);
19253
+ return sqlite3StrAccumFinish(&acc);
19254
+ }
19255
+ SQLITE_API char *sqlite3_snprintf(int n, char *zBuf, const char *zFormat, ...){
19256
+ char *z;
19257
+ va_list ap;
19258
+ va_start(ap,zFormat);
19259
+ z = sqlite3_vsnprintf(n, zBuf, zFormat, ap);
19176
19260
  va_end(ap);
19177
- z = sqlite3StrAccumFinish(&acc);
19178
19261
  return z;
19179
19262
  }
19180
19263
 
@@ -22189,7 +22272,7 @@ static int os2FileControl(sqlite3_file *id, int op, void *pArg){
22189
22272
  return SQLITE_OK;
22190
22273
  }
22191
22274
  }
22192
- return SQLITE_ERROR;
22275
+ return SQLITE_NOTFOUND;
22193
22276
  }
22194
22277
 
22195
22278
  /*
@@ -22905,7 +22988,9 @@ SQLITE_API int sqlite3_os_end(void){
22905
22988
  #include <unistd.h>
22906
22989
  #include <sys/time.h>
22907
22990
  #include <errno.h>
22991
+ #ifndef SQLITE_OMIT_WAL
22908
22992
  #include <sys/mman.h>
22993
+ #endif
22909
22994
 
22910
22995
  #if SQLITE_ENABLE_LOCKING_STYLE
22911
22996
  # include <sys/ioctl.h>
@@ -26123,8 +26208,11 @@ static int unixFileControl(sqlite3_file *id, int op, void *pArg){
26123
26208
  return proxyFileControl(id,op,pArg);
26124
26209
  }
26125
26210
  #endif /* SQLITE_ENABLE_LOCKING_STYLE && defined(__APPLE__) */
26211
+ case SQLITE_FCNTL_SYNC_OMITTED: {
26212
+ return SQLITE_OK; /* A no-op */
26213
+ }
26126
26214
  }
26127
- return SQLITE_ERROR;
26215
+ return SQLITE_NOTFOUND;
26128
26216
  }
26129
26217
 
26130
26218
  /*
@@ -28393,18 +28481,19 @@ extern int gethostuuid(uuid_t id, const struct timespec *wait);
28393
28481
  ** bytes of writable memory.
28394
28482
  */
28395
28483
  static int proxyGetHostID(unsigned char *pHostID, int *pError){
28396
- struct timespec timeout = {1, 0}; /* 1 sec timeout */
28397
-
28398
28484
  assert(PROXY_HOSTIDLEN == sizeof(uuid_t));
28399
28485
  memset(pHostID, 0, PROXY_HOSTIDLEN);
28400
28486
  #if defined(__MAX_OS_X_VERSION_MIN_REQUIRED)\
28401
28487
  && __MAC_OS_X_VERSION_MIN_REQUIRED<1050
28402
- if( gethostuuid(pHostID, &timeout) ){
28403
- int err = errno;
28404
- if( pError ){
28405
- *pError = err;
28488
+ {
28489
+ static const struct timespec timeout = {1, 0}; /* 1 sec timeout */
28490
+ if( gethostuuid(pHostID, &timeout) ){
28491
+ int err = errno;
28492
+ if( pError ){
28493
+ *pError = err;
28494
+ }
28495
+ return SQLITE_IOERR;
28406
28496
  }
28407
- return SQLITE_IOERR;
28408
28497
  }
28409
28498
  #endif
28410
28499
  #ifdef SQLITE_TEST
@@ -30710,8 +30799,11 @@ static int winFileControl(sqlite3_file *id, int op, void *pArg){
30710
30799
  SimulateIOErrorBenign(0);
30711
30800
  return SQLITE_OK;
30712
30801
  }
30802
+ case SQLITE_FCNTL_SYNC_OMITTED: {
30803
+ return SQLITE_OK;
30804
+ }
30713
30805
  }
30714
- return SQLITE_ERROR;
30806
+ return SQLITE_NOTFOUND;
30715
30807
  }
30716
30808
 
30717
30809
  /*
@@ -33331,6 +33423,38 @@ SQLITE_PRIVATE void sqlite3PcacheIterateDirty(PCache *pCache, void (*xIter)(PgHd
33331
33423
  typedef struct PCache1 PCache1;
33332
33424
  typedef struct PgHdr1 PgHdr1;
33333
33425
  typedef struct PgFreeslot PgFreeslot;
33426
+ typedef struct PGroup PGroup;
33427
+
33428
+ /* Each page cache (or PCache) belongs to a PGroup. A PGroup is a set
33429
+ ** of one or more PCaches that are able to recycle each others unpinned
33430
+ ** pages when they are under memory pressure. A PGroup is an instance of
33431
+ ** the following object.
33432
+ **
33433
+ ** This page cache implementation works in one of two modes:
33434
+ **
33435
+ ** (1) Every PCache is the sole member of its own PGroup. There is
33436
+ ** one PGroup per PCache.
33437
+ **
33438
+ ** (2) There is a single global PGroup that all PCaches are a member
33439
+ ** of.
33440
+ **
33441
+ ** Mode 1 uses more memory (since PCache instances are not able to rob
33442
+ ** unused pages from other PCaches) but it also operates without a mutex,
33443
+ ** and is therefore often faster. Mode 2 requires a mutex in order to be
33444
+ ** threadsafe, but is able recycle pages more efficient.
33445
+ **
33446
+ ** For mode (1), PGroup.mutex is NULL. For mode (2) there is only a single
33447
+ ** PGroup which is the pcache1.grp global variable and its mutex is
33448
+ ** SQLITE_MUTEX_STATIC_LRU.
33449
+ */
33450
+ struct PGroup {
33451
+ sqlite3_mutex *mutex; /* MUTEX_STATIC_LRU or NULL */
33452
+ int nMaxPage; /* Sum of nMax for purgeable caches */
33453
+ int nMinPage; /* Sum of nMin for purgeable caches */
33454
+ int mxPinned; /* nMaxpage + 10 - nMinPage */
33455
+ int nCurrentPage; /* Number of purgeable pages allocated */
33456
+ PgHdr1 *pLruHead, *pLruTail; /* LRU list of unpinned pages */
33457
+ };
33334
33458
 
33335
33459
  /* Each page cache is an instance of the following object. Every
33336
33460
  ** open database file (including each in-memory database and each
@@ -33344,16 +33468,17 @@ struct PCache1 {
33344
33468
  /* Cache configuration parameters. Page size (szPage) and the purgeable
33345
33469
  ** flag (bPurgeable) are set when the cache is created. nMax may be
33346
33470
  ** modified at any time by a call to the pcache1CacheSize() method.
33347
- ** The global mutex must be held when accessing nMax.
33471
+ ** The PGroup mutex must be held when accessing nMax.
33348
33472
  */
33473
+ PGroup *pGroup; /* PGroup this cache belongs to */
33349
33474
  int szPage; /* Size of allocated pages in bytes */
33350
33475
  int bPurgeable; /* True if cache is purgeable */
33351
33476
  unsigned int nMin; /* Minimum number of pages reserved */
33352
33477
  unsigned int nMax; /* Configured "cache_size" value */
33478
+ unsigned int n90pct; /* nMax*9/10 */
33353
33479
 
33354
33480
  /* Hash table of all pages. The following variables may only be accessed
33355
- ** when the accessor is holding the global mutex (see pcache1EnterMutex()
33356
- ** and pcache1LeaveMutex()).
33481
+ ** when the accessor is holding the PGroup mutex.
33357
33482
  */
33358
33483
  unsigned int nRecyclable; /* Number of pages in the LRU list */
33359
33484
  unsigned int nPage; /* Total number of pages in apHash */
@@ -33389,21 +33514,27 @@ struct PgFreeslot {
33389
33514
  ** Global data used by this cache.
33390
33515
  */
33391
33516
  static SQLITE_WSD struct PCacheGlobal {
33392
- sqlite3_mutex *mutex; /* static mutex MUTEX_STATIC_LRU */
33393
-
33394
- int nMaxPage; /* Sum of nMaxPage for purgeable caches */
33395
- int nMinPage; /* Sum of nMinPage for purgeable caches */
33396
- int nCurrentPage; /* Number of purgeable pages allocated */
33397
- PgHdr1 *pLruHead, *pLruTail; /* LRU list of unpinned pages */
33398
-
33399
- /* Variables related to SQLITE_CONFIG_PAGECACHE settings. */
33400
- int szSlot; /* Size of each free slot */
33401
- int nSlot; /* The number of pcache slots */
33402
- int nFreeSlot; /* Number of unused pcache slots */
33403
- int nReserve; /* Try to keep nFreeSlot above this */
33404
- void *pStart, *pEnd; /* Bounds of pagecache malloc range */
33405
- PgFreeslot *pFree; /* Free page blocks */
33406
- int isInit; /* True if initialized */
33517
+ PGroup grp; /* The global PGroup for mode (2) */
33518
+
33519
+ /* Variables related to SQLITE_CONFIG_PAGECACHE settings. The
33520
+ ** szSlot, nSlot, pStart, pEnd, nReserve, and isInit values are all
33521
+ ** fixed at sqlite3_initialize() time and do not require mutex protection.
33522
+ ** The nFreeSlot and pFree values do require mutex protection.
33523
+ */
33524
+ int isInit; /* True if initialized */
33525
+ int szSlot; /* Size of each free slot */
33526
+ int nSlot; /* The number of pcache slots */
33527
+ int nReserve; /* Try to keep nFreeSlot above this */
33528
+ void *pStart, *pEnd; /* Bounds of pagecache malloc range */
33529
+ /* Above requires no mutex. Use mutex below for variable that follow. */
33530
+ sqlite3_mutex *mutex; /* Mutex for accessing the following: */
33531
+ int nFreeSlot; /* Number of unused pcache slots */
33532
+ PgFreeslot *pFree; /* Free page blocks */
33533
+ /* The following value requires a mutex to change. We skip the mutex on
33534
+ ** reading because (1) most platforms read a 32-bit integer atomically and
33535
+ ** (2) even if an incorrect value is read, no great harm is done since this
33536
+ ** is really just an optimization. */
33537
+ int bUnderPressure; /* True if low on PAGECACHE memory */
33407
33538
  } pcache1_g;
33408
33539
 
33409
33540
  /*
@@ -33429,10 +33560,10 @@ static SQLITE_WSD struct PCacheGlobal {
33429
33560
  #define PAGE_TO_PGHDR1(c, p) (PgHdr1*)(((char*)p) + c->szPage)
33430
33561
 
33431
33562
  /*
33432
- ** Macros to enter and leave the global LRU mutex.
33563
+ ** Macros to enter and leave the PCache LRU mutex.
33433
33564
  */
33434
- #define pcache1EnterMutex() sqlite3_mutex_enter(pcache1.mutex)
33435
- #define pcache1LeaveMutex() sqlite3_mutex_leave(pcache1.mutex)
33565
+ #define pcache1EnterMutex(X) sqlite3_mutex_enter((X)->mutex)
33566
+ #define pcache1LeaveMutex(X) sqlite3_mutex_leave((X)->mutex)
33436
33567
 
33437
33568
  /******************************************************************************/
33438
33569
  /******** Page Allocation/SQLITE_CONFIG_PCACHE Related Functions **************/
@@ -33442,6 +33573,9 @@ static SQLITE_WSD struct PCacheGlobal {
33442
33573
  ** supplied to use for the page-cache by passing the SQLITE_CONFIG_PAGECACHE
33443
33574
  ** verb to sqlite3_config(). Parameter pBuf points to an allocation large
33444
33575
  ** enough to contain 'n' buffers of 'sz' bytes each.
33576
+ **
33577
+ ** This routine is called from sqlite3_initialize() and so it is guaranteed
33578
+ ** to be serialized already. There is no need for further mutexing.
33445
33579
  */
33446
33580
  SQLITE_PRIVATE void sqlite3PCacheBufferSetup(void *pBuf, int sz, int n){
33447
33581
  if( pcache1.isInit ){
@@ -33452,6 +33586,7 @@ SQLITE_PRIVATE void sqlite3PCacheBufferSetup(void *pBuf, int sz, int n){
33452
33586
  pcache1.nReserve = n>90 ? 10 : (n/10 + 1);
33453
33587
  pcache1.pStart = pBuf;
33454
33588
  pcache1.pFree = 0;
33589
+ pcache1.bUnderPressure = 0;
33455
33590
  while( n-- ){
33456
33591
  p = (PgFreeslot*)pBuf;
33457
33592
  p->pNext = pcache1.pFree;
@@ -33467,32 +33602,36 @@ SQLITE_PRIVATE void sqlite3PCacheBufferSetup(void *pBuf, int sz, int n){
33467
33602
  ** configured using sqlite3_config(SQLITE_CONFIG_PAGECACHE) option. If no
33468
33603
  ** such buffer exists or there is no space left in it, this function falls
33469
33604
  ** back to sqlite3Malloc().
33605
+ **
33606
+ ** Multiple threads can run this routine at the same time. Global variables
33607
+ ** in pcache1 need to be protected via mutex.
33470
33608
  */
33471
33609
  static void *pcache1Alloc(int nByte){
33472
- void *p;
33473
- assert( sqlite3_mutex_held(pcache1.mutex) );
33610
+ void *p = 0;
33611
+ assert( sqlite3_mutex_notheld(pcache1.grp.mutex) );
33474
33612
  sqlite3StatusSet(SQLITE_STATUS_PAGECACHE_SIZE, nByte);
33475
- if( nByte<=pcache1.szSlot && pcache1.pFree ){
33476
- assert( pcache1.isInit );
33613
+ if( nByte<=pcache1.szSlot ){
33614
+ sqlite3_mutex_enter(pcache1.mutex);
33477
33615
  p = (PgHdr1 *)pcache1.pFree;
33478
- pcache1.pFree = pcache1.pFree->pNext;
33479
- pcache1.nFreeSlot--;
33480
- assert( pcache1.nFreeSlot>=0 );
33481
- sqlite3StatusAdd(SQLITE_STATUS_PAGECACHE_USED, 1);
33482
- }else{
33483
-
33484
- /* Allocate a new buffer using sqlite3Malloc. Before doing so, exit the
33485
- ** global pcache mutex and unlock the pager-cache object pCache. This is
33486
- ** so that if the attempt to allocate a new buffer causes the the
33487
- ** configured soft-heap-limit to be breached, it will be possible to
33488
- ** reclaim memory from this pager-cache.
33616
+ if( p ){
33617
+ pcache1.pFree = pcache1.pFree->pNext;
33618
+ pcache1.nFreeSlot--;
33619
+ pcache1.bUnderPressure = pcache1.nFreeSlot<pcache1.nReserve;
33620
+ assert( pcache1.nFreeSlot>=0 );
33621
+ sqlite3StatusAdd(SQLITE_STATUS_PAGECACHE_USED, 1);
33622
+ }
33623
+ sqlite3_mutex_leave(pcache1.mutex);
33624
+ }
33625
+ if( p==0 ){
33626
+ /* Memory is not available in the SQLITE_CONFIG_PAGECACHE pool. Get
33627
+ ** it from sqlite3Malloc instead.
33489
33628
  */
33490
- pcache1LeaveMutex();
33491
33629
  p = sqlite3Malloc(nByte);
33492
- pcache1EnterMutex();
33493
33630
  if( p ){
33494
33631
  int sz = sqlite3MallocSize(p);
33632
+ sqlite3_mutex_enter(pcache1.mutex);
33495
33633
  sqlite3StatusAdd(SQLITE_STATUS_PAGECACHE_OVERFLOW, sz);
33634
+ sqlite3_mutex_leave(pcache1.mutex);
33496
33635
  }
33497
33636
  sqlite3MemdebugSetType(p, MEMTYPE_PCACHE);
33498
33637
  }
@@ -33503,22 +33642,26 @@ static void *pcache1Alloc(int nByte){
33503
33642
  ** Free an allocated buffer obtained from pcache1Alloc().
33504
33643
  */
33505
33644
  static void pcache1Free(void *p){
33506
- assert( sqlite3_mutex_held(pcache1.mutex) );
33507
33645
  if( p==0 ) return;
33508
33646
  if( p>=pcache1.pStart && p<pcache1.pEnd ){
33509
33647
  PgFreeslot *pSlot;
33648
+ sqlite3_mutex_enter(pcache1.mutex);
33510
33649
  sqlite3StatusAdd(SQLITE_STATUS_PAGECACHE_USED, -1);
33511
33650
  pSlot = (PgFreeslot*)p;
33512
33651
  pSlot->pNext = pcache1.pFree;
33513
33652
  pcache1.pFree = pSlot;
33514
33653
  pcache1.nFreeSlot++;
33654
+ pcache1.bUnderPressure = pcache1.nFreeSlot<pcache1.nReserve;
33515
33655
  assert( pcache1.nFreeSlot<=pcache1.nSlot );
33656
+ sqlite3_mutex_leave(pcache1.mutex);
33516
33657
  }else{
33517
33658
  int iSize;
33518
33659
  assert( sqlite3MemdebugHasType(p, MEMTYPE_PCACHE) );
33519
33660
  sqlite3MemdebugSetType(p, MEMTYPE_HEAP);
33520
33661
  iSize = sqlite3MallocSize(p);
33662
+ sqlite3_mutex_enter(pcache1.mutex);
33521
33663
  sqlite3StatusAdd(SQLITE_STATUS_PAGECACHE_OVERFLOW, -iSize);
33664
+ sqlite3_mutex_leave(pcache1.mutex);
33522
33665
  sqlite3_free(p);
33523
33666
  }
33524
33667
  }
@@ -33528,7 +33671,6 @@ static void pcache1Free(void *p){
33528
33671
  ** Return the size of a pcache allocation
33529
33672
  */
33530
33673
  static int pcache1MemSize(void *p){
33531
- assert( sqlite3_mutex_held(pcache1.mutex) );
33532
33674
  if( p>=pcache1.pStart && p<pcache1.pEnd ){
33533
33675
  return pcache1.szSlot;
33534
33676
  }else{
@@ -33552,7 +33694,7 @@ static PgHdr1 *pcache1AllocPage(PCache1 *pCache){
33552
33694
  if( pPg ){
33553
33695
  p = PAGE_TO_PGHDR1(pCache, pPg);
33554
33696
  if( pCache->bPurgeable ){
33555
- pcache1.nCurrentPage++;
33697
+ pCache->pGroup->nCurrentPage++;
33556
33698
  }
33557
33699
  }else{
33558
33700
  p = 0;
@@ -33569,8 +33711,9 @@ static PgHdr1 *pcache1AllocPage(PCache1 *pCache){
33569
33711
  */
33570
33712
  static void pcache1FreePage(PgHdr1 *p){
33571
33713
  if( ALWAYS(p) ){
33572
- if( p->pCache->bPurgeable ){
33573
- pcache1.nCurrentPage--;
33714
+ PCache1 *pCache = p->pCache;
33715
+ if( pCache->bPurgeable ){
33716
+ pCache->pGroup->nCurrentPage--;
33574
33717
  }
33575
33718
  pcache1Free(PGHDR1_TO_PAGE(p));
33576
33719
  }
@@ -33582,20 +33725,14 @@ static void pcache1FreePage(PgHdr1 *p){
33582
33725
  ** exists, this function falls back to sqlite3Malloc().
33583
33726
  */
33584
33727
  SQLITE_PRIVATE void *sqlite3PageMalloc(int sz){
33585
- void *p;
33586
- pcache1EnterMutex();
33587
- p = pcache1Alloc(sz);
33588
- pcache1LeaveMutex();
33589
- return p;
33728
+ return pcache1Alloc(sz);
33590
33729
  }
33591
33730
 
33592
33731
  /*
33593
33732
  ** Free an allocated buffer obtained from sqlite3PageMalloc().
33594
33733
  */
33595
33734
  SQLITE_PRIVATE void sqlite3PageFree(void *p){
33596
- pcache1EnterMutex();
33597
33735
  pcache1Free(p);
33598
- pcache1LeaveMutex();
33599
33736
  }
33600
33737
 
33601
33738
 
@@ -33616,9 +33753,8 @@ SQLITE_PRIVATE void sqlite3PageFree(void *p){
33616
33753
  ** the heap even further.
33617
33754
  */
33618
33755
  static int pcache1UnderMemoryPressure(PCache1 *pCache){
33619
- assert( sqlite3_mutex_held(pcache1.mutex) );
33620
33756
  if( pcache1.nSlot && pCache->szPage<=pcache1.szSlot ){
33621
- return pcache1.nFreeSlot<pcache1.nReserve;
33757
+ return pcache1.bUnderPressure;
33622
33758
  }else{
33623
33759
  return sqlite3HeapNearlyFull();
33624
33760
  }
@@ -33631,25 +33767,25 @@ static int pcache1UnderMemoryPressure(PCache1 *pCache){
33631
33767
  ** This function is used to resize the hash table used by the cache passed
33632
33768
  ** as the first argument.
33633
33769
  **
33634
- ** The global mutex must be held when this function is called.
33770
+ ** The PCache mutex must be held when this function is called.
33635
33771
  */
33636
33772
  static int pcache1ResizeHash(PCache1 *p){
33637
33773
  PgHdr1 **apNew;
33638
33774
  unsigned int nNew;
33639
33775
  unsigned int i;
33640
33776
 
33641
- assert( sqlite3_mutex_held(pcache1.mutex) );
33777
+ assert( sqlite3_mutex_held(p->pGroup->mutex) );
33642
33778
 
33643
33779
  nNew = p->nHash*2;
33644
33780
  if( nNew<256 ){
33645
33781
  nNew = 256;
33646
33782
  }
33647
33783
 
33648
- pcache1LeaveMutex();
33784
+ pcache1LeaveMutex(p->pGroup);
33649
33785
  if( p->nHash ){ sqlite3BeginBenignMalloc(); }
33650
33786
  apNew = (PgHdr1 **)sqlite3_malloc(sizeof(PgHdr1 *)*nNew);
33651
33787
  if( p->nHash ){ sqlite3EndBenignMalloc(); }
33652
- pcache1EnterMutex();
33788
+ pcache1EnterMutex(p->pGroup);
33653
33789
  if( apNew ){
33654
33790
  memset(apNew, 0, sizeof(PgHdr1 *)*nNew);
33655
33791
  for(i=0; i<p->nHash; i++){
@@ -33672,25 +33808,33 @@ static int pcache1ResizeHash(PCache1 *p){
33672
33808
 
33673
33809
  /*
33674
33810
  ** This function is used internally to remove the page pPage from the
33675
- ** global LRU list, if is part of it. If pPage is not part of the global
33811
+ ** PGroup LRU list, if is part of it. If pPage is not part of the PGroup
33676
33812
  ** LRU list, then this function is a no-op.
33677
33813
  **
33678
- ** The global mutex must be held when this function is called.
33814
+ ** The PGroup mutex must be held when this function is called.
33815
+ **
33816
+ ** If pPage is NULL then this routine is a no-op.
33679
33817
  */
33680
33818
  static void pcache1PinPage(PgHdr1 *pPage){
33681
- assert( sqlite3_mutex_held(pcache1.mutex) );
33682
- if( pPage && (pPage->pLruNext || pPage==pcache1.pLruTail) ){
33819
+ PCache1 *pCache;
33820
+ PGroup *pGroup;
33821
+
33822
+ if( pPage==0 ) return;
33823
+ pCache = pPage->pCache;
33824
+ pGroup = pCache->pGroup;
33825
+ assert( sqlite3_mutex_held(pGroup->mutex) );
33826
+ if( pPage->pLruNext || pPage==pGroup->pLruTail ){
33683
33827
  if( pPage->pLruPrev ){
33684
33828
  pPage->pLruPrev->pLruNext = pPage->pLruNext;
33685
33829
  }
33686
33830
  if( pPage->pLruNext ){
33687
33831
  pPage->pLruNext->pLruPrev = pPage->pLruPrev;
33688
33832
  }
33689
- if( pcache1.pLruHead==pPage ){
33690
- pcache1.pLruHead = pPage->pLruNext;
33833
+ if( pGroup->pLruHead==pPage ){
33834
+ pGroup->pLruHead = pPage->pLruNext;
33691
33835
  }
33692
- if( pcache1.pLruTail==pPage ){
33693
- pcache1.pLruTail = pPage->pLruPrev;
33836
+ if( pGroup->pLruTail==pPage ){
33837
+ pGroup->pLruTail = pPage->pLruPrev;
33694
33838
  }
33695
33839
  pPage->pLruNext = 0;
33696
33840
  pPage->pLruPrev = 0;
@@ -33703,13 +33847,14 @@ static void pcache1PinPage(PgHdr1 *pPage){
33703
33847
  ** Remove the page supplied as an argument from the hash table
33704
33848
  ** (PCache1.apHash structure) that it is currently stored in.
33705
33849
  **
33706
- ** The global mutex must be held when this function is called.
33850
+ ** The PGroup mutex must be held when this function is called.
33707
33851
  */
33708
33852
  static void pcache1RemoveFromHash(PgHdr1 *pPage){
33709
33853
  unsigned int h;
33710
33854
  PCache1 *pCache = pPage->pCache;
33711
33855
  PgHdr1 **pp;
33712
33856
 
33857
+ assert( sqlite3_mutex_held(pCache->pGroup->mutex) );
33713
33858
  h = pPage->iKey % pCache->nHash;
33714
33859
  for(pp=&pCache->apHash[h]; (*pp)!=pPage; pp=&(*pp)->pNext);
33715
33860
  *pp = (*pp)->pNext;
@@ -33718,13 +33863,14 @@ static void pcache1RemoveFromHash(PgHdr1 *pPage){
33718
33863
  }
33719
33864
 
33720
33865
  /*
33721
- ** If there are currently more than pcache.nMaxPage pages allocated, try
33722
- ** to recycle pages to reduce the number allocated to pcache.nMaxPage.
33866
+ ** If there are currently more than nMaxPage pages allocated, try
33867
+ ** to recycle pages to reduce the number allocated to nMaxPage.
33723
33868
  */
33724
- static void pcache1EnforceMaxPage(void){
33725
- assert( sqlite3_mutex_held(pcache1.mutex) );
33726
- while( pcache1.nCurrentPage>pcache1.nMaxPage && pcache1.pLruTail ){
33727
- PgHdr1 *p = pcache1.pLruTail;
33869
+ static void pcache1EnforceMaxPage(PGroup *pGroup){
33870
+ assert( sqlite3_mutex_held(pGroup->mutex) );
33871
+ while( pGroup->nCurrentPage>pGroup->nMaxPage && pGroup->pLruTail ){
33872
+ PgHdr1 *p = pGroup->pLruTail;
33873
+ assert( p->pCache->pGroup==pGroup );
33728
33874
  pcache1PinPage(p);
33729
33875
  pcache1RemoveFromHash(p);
33730
33876
  pcache1FreePage(p);
@@ -33736,15 +33882,15 @@ static void pcache1EnforceMaxPage(void){
33736
33882
  ** greater than or equal to iLimit. Any pinned pages that meet this
33737
33883
  ** criteria are unpinned before they are discarded.
33738
33884
  **
33739
- ** The global mutex must be held when this function is called.
33885
+ ** The PCache mutex must be held when this function is called.
33740
33886
  */
33741
33887
  static void pcache1TruncateUnsafe(
33742
- PCache1 *pCache,
33743
- unsigned int iLimit
33888
+ PCache1 *pCache, /* The cache to truncate */
33889
+ unsigned int iLimit /* Drop pages with this pgno or larger */
33744
33890
  ){
33745
- TESTONLY( unsigned int nPage = 0; ) /* Used to assert pCache->nPage is correct */
33891
+ TESTONLY( unsigned int nPage = 0; ) /* To assert pCache->nPage is correct */
33746
33892
  unsigned int h;
33747
- assert( sqlite3_mutex_held(pcache1.mutex) );
33893
+ assert( sqlite3_mutex_held(pCache->pGroup->mutex) );
33748
33894
  for(h=0; h<pCache->nHash; h++){
33749
33895
  PgHdr1 **pp = &pCache->apHash[h];
33750
33896
  PgHdr1 *pPage;
@@ -33774,8 +33920,10 @@ static int pcache1Init(void *NotUsed){
33774
33920
  assert( pcache1.isInit==0 );
33775
33921
  memset(&pcache1, 0, sizeof(pcache1));
33776
33922
  if( sqlite3GlobalConfig.bCoreMutex ){
33777
- pcache1.mutex = sqlite3_mutex_alloc(SQLITE_MUTEX_STATIC_LRU);
33923
+ pcache1.grp.mutex = sqlite3_mutex_alloc(SQLITE_MUTEX_STATIC_LRU);
33924
+ pcache1.mutex = sqlite3_mutex_alloc(SQLITE_MUTEX_STATIC_PMEM);
33778
33925
  }
33926
+ pcache1.grp.mxPinned = 10;
33779
33927
  pcache1.isInit = 1;
33780
33928
  return SQLITE_OK;
33781
33929
  }
@@ -33797,18 +33945,47 @@ static void pcache1Shutdown(void *NotUsed){
33797
33945
  ** Allocate a new cache.
33798
33946
  */
33799
33947
  static sqlite3_pcache *pcache1Create(int szPage, int bPurgeable){
33800
- PCache1 *pCache;
33948
+ PCache1 *pCache; /* The newly created page cache */
33949
+ PGroup *pGroup; /* The group the new page cache will belong to */
33950
+ int sz; /* Bytes of memory required to allocate the new cache */
33801
33951
 
33802
- pCache = (PCache1 *)sqlite3_malloc(sizeof(PCache1));
33952
+ /*
33953
+ ** The seperateCache variable is true if each PCache has its own private
33954
+ ** PGroup. In other words, separateCache is true for mode (1) where no
33955
+ ** mutexing is required.
33956
+ **
33957
+ ** * Always use a unified cache (mode-2) if ENABLE_MEMORY_MANAGEMENT
33958
+ **
33959
+ ** * Always use a unified cache in single-threaded applications
33960
+ **
33961
+ ** * Otherwise (if multi-threaded and ENABLE_MEMORY_MANAGEMENT is off)
33962
+ ** use separate caches (mode-1)
33963
+ */
33964
+ #if defined(SQLITE_ENABLE_MEMORY_MANAGEMENT) || SQLITE_THREADSAFE==0
33965
+ const int separateCache = 0;
33966
+ #else
33967
+ int separateCache = sqlite3GlobalConfig.bCoreMutex>0;
33968
+ #endif
33969
+
33970
+ sz = sizeof(PCache1) + sizeof(PGroup)*separateCache;
33971
+ pCache = (PCache1 *)sqlite3_malloc(sz);
33803
33972
  if( pCache ){
33804
- memset(pCache, 0, sizeof(PCache1));
33973
+ memset(pCache, 0, sz);
33974
+ if( separateCache ){
33975
+ pGroup = (PGroup*)&pCache[1];
33976
+ pGroup->mxPinned = 10;
33977
+ }else{
33978
+ pGroup = &pcache1_g.grp;
33979
+ }
33980
+ pCache->pGroup = pGroup;
33805
33981
  pCache->szPage = szPage;
33806
33982
  pCache->bPurgeable = (bPurgeable ? 1 : 0);
33807
33983
  if( bPurgeable ){
33808
33984
  pCache->nMin = 10;
33809
- pcache1EnterMutex();
33810
- pcache1.nMinPage += pCache->nMin;
33811
- pcache1LeaveMutex();
33985
+ pcache1EnterMutex(pGroup);
33986
+ pGroup->nMinPage += pCache->nMin;
33987
+ pGroup->mxPinned = pGroup->nMaxPage + 10 - pGroup->nMinPage;
33988
+ pcache1LeaveMutex(pGroup);
33812
33989
  }
33813
33990
  }
33814
33991
  return (sqlite3_pcache *)pCache;
@@ -33822,11 +33999,14 @@ static sqlite3_pcache *pcache1Create(int szPage, int bPurgeable){
33822
33999
  static void pcache1Cachesize(sqlite3_pcache *p, int nMax){
33823
34000
  PCache1 *pCache = (PCache1 *)p;
33824
34001
  if( pCache->bPurgeable ){
33825
- pcache1EnterMutex();
33826
- pcache1.nMaxPage += (nMax - pCache->nMax);
34002
+ PGroup *pGroup = pCache->pGroup;
34003
+ pcache1EnterMutex(pGroup);
34004
+ pGroup->nMaxPage += (nMax - pCache->nMax);
34005
+ pGroup->mxPinned = pGroup->nMaxPage + 10 - pGroup->nMinPage;
33827
34006
  pCache->nMax = nMax;
33828
- pcache1EnforceMaxPage();
33829
- pcache1LeaveMutex();
34007
+ pCache->n90pct = pCache->nMax*9/10;
34008
+ pcache1EnforceMaxPage(pGroup);
34009
+ pcache1LeaveMutex(pGroup);
33830
34010
  }
33831
34011
  }
33832
34012
 
@@ -33835,9 +34015,10 @@ static void pcache1Cachesize(sqlite3_pcache *p, int nMax){
33835
34015
  */
33836
34016
  static int pcache1Pagecount(sqlite3_pcache *p){
33837
34017
  int n;
33838
- pcache1EnterMutex();
33839
- n = ((PCache1 *)p)->nPage;
33840
- pcache1LeaveMutex();
34018
+ PCache1 *pCache = (PCache1*)p;
34019
+ pcache1EnterMutex(pCache->pGroup);
34020
+ n = pCache->nPage;
34021
+ pcache1LeaveMutex(pCache->pGroup);
33841
34022
  return n;
33842
34023
  }
33843
34024
 
@@ -33896,30 +34077,49 @@ static int pcache1Pagecount(sqlite3_pcache *p){
33896
34077
  ** 5. Otherwise, allocate and return a new page buffer.
33897
34078
  */
33898
34079
  static void *pcache1Fetch(sqlite3_pcache *p, unsigned int iKey, int createFlag){
33899
- unsigned int nPinned;
34080
+ int nPinned;
33900
34081
  PCache1 *pCache = (PCache1 *)p;
34082
+ PGroup *pGroup;
33901
34083
  PgHdr1 *pPage = 0;
33902
34084
 
33903
34085
  assert( pCache->bPurgeable || createFlag!=1 );
33904
- pcache1EnterMutex();
33905
- if( createFlag==1 ) sqlite3BeginBenignMalloc();
34086
+ assert( pCache->bPurgeable || pCache->nMin==0 );
34087
+ assert( pCache->bPurgeable==0 || pCache->nMin==10 );
34088
+ assert( pCache->nMin==0 || pCache->bPurgeable );
34089
+ pcache1EnterMutex(pGroup = pCache->pGroup);
33906
34090
 
33907
- /* Search the hash table for an existing entry. */
34091
+ /* Step 1: Search the hash table for an existing entry. */
33908
34092
  if( pCache->nHash>0 ){
33909
34093
  unsigned int h = iKey % pCache->nHash;
33910
34094
  for(pPage=pCache->apHash[h]; pPage&&pPage->iKey!=iKey; pPage=pPage->pNext);
33911
34095
  }
33912
34096
 
34097
+ /* Step 2: Abort if no existing page is found and createFlag is 0 */
33913
34098
  if( pPage || createFlag==0 ){
33914
34099
  pcache1PinPage(pPage);
33915
34100
  goto fetch_out;
33916
34101
  }
33917
34102
 
33918
- /* Step 3 of header comment. */
34103
+ /* The pGroup local variable will normally be initialized by the
34104
+ ** pcache1EnterMutex() macro above. But if SQLITE_MUTEX_OMIT is defined,
34105
+ ** then pcache1EnterMutex() is a no-op, so we have to initialize the
34106
+ ** local variable here. Delaying the initialization of pGroup is an
34107
+ ** optimization: The common case is to exit the module before reaching
34108
+ ** this point.
34109
+ */
34110
+ #ifdef SQLITE_MUTEX_OMIT
34111
+ pGroup = pCache->pGroup;
34112
+ #endif
34113
+
34114
+
34115
+ /* Step 3: Abort if createFlag is 1 but the cache is nearly full */
33919
34116
  nPinned = pCache->nPage - pCache->nRecyclable;
34117
+ assert( nPinned>=0 );
34118
+ assert( pGroup->mxPinned == pGroup->nMaxPage + 10 - pGroup->nMinPage );
34119
+ assert( pCache->n90pct == pCache->nMax*9/10 );
33920
34120
  if( createFlag==1 && (
33921
- nPinned>=(pcache1.nMaxPage+pCache->nMin-pcache1.nMinPage)
33922
- || nPinned>=(pCache->nMax * 9 / 10)
34121
+ nPinned>=pGroup->mxPinned
34122
+ || nPinned>=(int)pCache->n90pct
33923
34123
  || pcache1UnderMemoryPressure(pCache)
33924
34124
  )){
33925
34125
  goto fetch_out;
@@ -33929,20 +34129,22 @@ static void *pcache1Fetch(sqlite3_pcache *p, unsigned int iKey, int createFlag){
33929
34129
  goto fetch_out;
33930
34130
  }
33931
34131
 
33932
- /* Step 4. Try to recycle a page buffer if appropriate. */
33933
- if( pCache->bPurgeable && pcache1.pLruTail && (
34132
+ /* Step 4. Try to recycle a page. */
34133
+ if( pCache->bPurgeable && pGroup->pLruTail && (
33934
34134
  (pCache->nPage+1>=pCache->nMax)
33935
- || pcache1.nCurrentPage>=pcache1.nMaxPage
34135
+ || pGroup->nCurrentPage>=pGroup->nMaxPage
33936
34136
  || pcache1UnderMemoryPressure(pCache)
33937
34137
  )){
33938
- pPage = pcache1.pLruTail;
34138
+ PCache1 *pOtherCache;
34139
+ pPage = pGroup->pLruTail;
33939
34140
  pcache1RemoveFromHash(pPage);
33940
34141
  pcache1PinPage(pPage);
33941
- if( pPage->pCache->szPage!=pCache->szPage ){
34142
+ if( (pOtherCache = pPage->pCache)->szPage!=pCache->szPage ){
33942
34143
  pcache1FreePage(pPage);
33943
34144
  pPage = 0;
33944
34145
  }else{
33945
- pcache1.nCurrentPage -= (pPage->pCache->bPurgeable - pCache->bPurgeable);
34146
+ pGroup->nCurrentPage -=
34147
+ (pOtherCache->bPurgeable - pCache->bPurgeable);
33946
34148
  }
33947
34149
  }
33948
34150
 
@@ -33950,7 +34152,11 @@ static void *pcache1Fetch(sqlite3_pcache *p, unsigned int iKey, int createFlag){
33950
34152
  ** attempt to allocate a new one.
33951
34153
  */
33952
34154
  if( !pPage ){
34155
+ if( createFlag==1 ) sqlite3BeginBenignMalloc();
34156
+ pcache1LeaveMutex(pGroup);
33953
34157
  pPage = pcache1AllocPage(pCache);
34158
+ pcache1EnterMutex(pGroup);
34159
+ if( createFlag==1 ) sqlite3EndBenignMalloc();
33954
34160
  }
33955
34161
 
33956
34162
  if( pPage ){
@@ -33969,8 +34175,7 @@ fetch_out:
33969
34175
  if( pPage && iKey>pCache->iMaxKey ){
33970
34176
  pCache->iMaxKey = iKey;
33971
34177
  }
33972
- if( createFlag==1 ) sqlite3EndBenignMalloc();
33973
- pcache1LeaveMutex();
34178
+ pcache1LeaveMutex(pGroup);
33974
34179
  return (pPage ? PGHDR1_TO_PAGE(pPage) : 0);
33975
34180
  }
33976
34181
 
@@ -33983,37 +34188,34 @@ fetch_out:
33983
34188
  static void pcache1Unpin(sqlite3_pcache *p, void *pPg, int reuseUnlikely){
33984
34189
  PCache1 *pCache = (PCache1 *)p;
33985
34190
  PgHdr1 *pPage = PAGE_TO_PGHDR1(pCache, pPg);
34191
+ PGroup *pGroup = pCache->pGroup;
33986
34192
 
33987
34193
  assert( pPage->pCache==pCache );
33988
- pcache1EnterMutex();
34194
+ pcache1EnterMutex(pGroup);
33989
34195
 
33990
34196
  /* It is an error to call this function if the page is already
33991
- ** part of the global LRU list.
34197
+ ** part of the PGroup LRU list.
33992
34198
  */
33993
34199
  assert( pPage->pLruPrev==0 && pPage->pLruNext==0 );
33994
- assert( pcache1.pLruHead!=pPage && pcache1.pLruTail!=pPage );
34200
+ assert( pGroup->pLruHead!=pPage && pGroup->pLruTail!=pPage );
33995
34201
 
33996
- if( reuseUnlikely || pcache1.nCurrentPage>pcache1.nMaxPage ){
34202
+ if( reuseUnlikely || pGroup->nCurrentPage>pGroup->nMaxPage ){
33997
34203
  pcache1RemoveFromHash(pPage);
33998
34204
  pcache1FreePage(pPage);
33999
34205
  }else{
34000
- /* Add the page to the global LRU list. Normally, the page is added to
34001
- ** the head of the list (last page to be recycled). However, if the
34002
- ** reuseUnlikely flag passed to this function is true, the page is added
34003
- ** to the tail of the list (first page to be recycled).
34004
- */
34005
- if( pcache1.pLruHead ){
34006
- pcache1.pLruHead->pLruPrev = pPage;
34007
- pPage->pLruNext = pcache1.pLruHead;
34008
- pcache1.pLruHead = pPage;
34206
+ /* Add the page to the PGroup LRU list. */
34207
+ if( pGroup->pLruHead ){
34208
+ pGroup->pLruHead->pLruPrev = pPage;
34209
+ pPage->pLruNext = pGroup->pLruHead;
34210
+ pGroup->pLruHead = pPage;
34009
34211
  }else{
34010
- pcache1.pLruTail = pPage;
34011
- pcache1.pLruHead = pPage;
34212
+ pGroup->pLruTail = pPage;
34213
+ pGroup->pLruHead = pPage;
34012
34214
  }
34013
34215
  pCache->nRecyclable++;
34014
34216
  }
34015
34217
 
34016
- pcache1LeaveMutex();
34218
+ pcache1LeaveMutex(pCache->pGroup);
34017
34219
  }
34018
34220
 
34019
34221
  /*
@@ -34032,7 +34234,7 @@ static void pcache1Rekey(
34032
34234
  assert( pPage->iKey==iOld );
34033
34235
  assert( pPage->pCache==pCache );
34034
34236
 
34035
- pcache1EnterMutex();
34237
+ pcache1EnterMutex(pCache->pGroup);
34036
34238
 
34037
34239
  h = iOld%pCache->nHash;
34038
34240
  pp = &pCache->apHash[h];
@@ -34049,7 +34251,7 @@ static void pcache1Rekey(
34049
34251
  pCache->iMaxKey = iNew;
34050
34252
  }
34051
34253
 
34052
- pcache1LeaveMutex();
34254
+ pcache1LeaveMutex(pCache->pGroup);
34053
34255
  }
34054
34256
 
34055
34257
  /*
@@ -34061,12 +34263,12 @@ static void pcache1Rekey(
34061
34263
  */
34062
34264
  static void pcache1Truncate(sqlite3_pcache *p, unsigned int iLimit){
34063
34265
  PCache1 *pCache = (PCache1 *)p;
34064
- pcache1EnterMutex();
34266
+ pcache1EnterMutex(pCache->pGroup);
34065
34267
  if( iLimit<=pCache->iMaxKey ){
34066
34268
  pcache1TruncateUnsafe(pCache, iLimit);
34067
34269
  pCache->iMaxKey = iLimit-1;
34068
34270
  }
34069
- pcache1LeaveMutex();
34271
+ pcache1LeaveMutex(pCache->pGroup);
34070
34272
  }
34071
34273
 
34072
34274
  /*
@@ -34076,13 +34278,15 @@ static void pcache1Truncate(sqlite3_pcache *p, unsigned int iLimit){
34076
34278
  */
34077
34279
  static void pcache1Destroy(sqlite3_pcache *p){
34078
34280
  PCache1 *pCache = (PCache1 *)p;
34281
+ PGroup *pGroup = pCache->pGroup;
34079
34282
  assert( pCache->bPurgeable || (pCache->nMax==0 && pCache->nMin==0) );
34080
- pcache1EnterMutex();
34283
+ pcache1EnterMutex(pGroup);
34081
34284
  pcache1TruncateUnsafe(pCache, 0);
34082
- pcache1.nMaxPage -= pCache->nMax;
34083
- pcache1.nMinPage -= pCache->nMin;
34084
- pcache1EnforceMaxPage();
34085
- pcache1LeaveMutex();
34285
+ pGroup->nMaxPage -= pCache->nMax;
34286
+ pGroup->nMinPage -= pCache->nMin;
34287
+ pGroup->mxPinned = pGroup->nMaxPage + 10 - pGroup->nMinPage;
34288
+ pcache1EnforceMaxPage(pGroup);
34289
+ pcache1LeaveMutex(pGroup);
34086
34290
  sqlite3_free(pCache->apHash);
34087
34291
  sqlite3_free(pCache);
34088
34292
  }
@@ -34121,16 +34325,18 @@ SQLITE_PRIVATE void sqlite3PCacheSetDefault(void){
34121
34325
  */
34122
34326
  SQLITE_PRIVATE int sqlite3PcacheReleaseMemory(int nReq){
34123
34327
  int nFree = 0;
34328
+ assert( sqlite3_mutex_notheld(pcache1.grp.mutex) );
34329
+ assert( sqlite3_mutex_notheld(pcache1.mutex) );
34124
34330
  if( pcache1.pStart==0 ){
34125
34331
  PgHdr1 *p;
34126
- pcache1EnterMutex();
34127
- while( (nReq<0 || nFree<nReq) && ((p=pcache1.pLruTail)!=0) ){
34332
+ pcache1EnterMutex(&pcache1.grp);
34333
+ while( (nReq<0 || nFree<nReq) && ((p=pcache1.grp.pLruTail)!=0) ){
34128
34334
  nFree += pcache1MemSize(PGHDR1_TO_PAGE(p));
34129
34335
  pcache1PinPage(p);
34130
34336
  pcache1RemoveFromHash(p);
34131
34337
  pcache1FreePage(p);
34132
34338
  }
34133
- pcache1LeaveMutex();
34339
+ pcache1LeaveMutex(&pcache1.grp);
34134
34340
  }
34135
34341
  return nFree;
34136
34342
  }
@@ -34149,12 +34355,12 @@ SQLITE_PRIVATE void sqlite3PcacheStats(
34149
34355
  ){
34150
34356
  PgHdr1 *p;
34151
34357
  int nRecyclable = 0;
34152
- for(p=pcache1.pLruHead; p; p=p->pLruNext){
34358
+ for(p=pcache1.grp.pLruHead; p; p=p->pLruNext){
34153
34359
  nRecyclable++;
34154
34360
  }
34155
- *pnCurrent = pcache1.nCurrentPage;
34156
- *pnMax = pcache1.nMaxPage;
34157
- *pnMin = pcache1.nMinPage;
34361
+ *pnCurrent = pcache1.grp.nCurrentPage;
34362
+ *pnMax = pcache1.grp.nMaxPage;
34363
+ *pnMin = pcache1.grp.nMinPage;
34158
34364
  *pnRecyclable = nRecyclable;
34159
34365
  }
34160
34366
  #endif
@@ -37179,15 +37385,21 @@ static int pager_truncate(Pager *pPager, Pgno nPage){
37179
37385
  && (pPager->eState>=PAGER_WRITER_DBMOD || pPager->eState==PAGER_OPEN)
37180
37386
  ){
37181
37387
  i64 currentSize, newSize;
37388
+ int szPage = pPager->pageSize;
37182
37389
  assert( pPager->eLock==EXCLUSIVE_LOCK );
37183
37390
  /* TODO: Is it safe to use Pager.dbFileSize here? */
37184
37391
  rc = sqlite3OsFileSize(pPager->fd, &currentSize);
37185
- newSize = pPager->pageSize*(i64)nPage;
37392
+ newSize = szPage*(i64)nPage;
37186
37393
  if( rc==SQLITE_OK && currentSize!=newSize ){
37187
37394
  if( currentSize>newSize ){
37188
37395
  rc = sqlite3OsTruncate(pPager->fd, newSize);
37189
37396
  }else{
37190
- rc = sqlite3OsWrite(pPager->fd, "", 1, newSize-1);
37397
+ char *pTmp = pPager->pTmpSpace;
37398
+ memset(pTmp, 0, szPage);
37399
+ testcase( (newSize-szPage) < currentSize );
37400
+ testcase( (newSize-szPage) == currentSize );
37401
+ testcase( (newSize-szPage) > currentSize );
37402
+ rc = sqlite3OsWrite(pPager->fd, pTmp, szPage, newSize-szPage);
37191
37403
  }
37192
37404
  if( rc==SQLITE_OK ){
37193
37405
  pPager->dbFileSize = nPage;
@@ -37451,10 +37663,10 @@ end_playback:
37451
37663
  rc = readMasterJournal(pPager->jfd, zMaster, pPager->pVfs->mxPathname+1);
37452
37664
  testcase( rc!=SQLITE_OK );
37453
37665
  }
37454
- if( rc==SQLITE_OK && !pPager->noSync
37666
+ if( rc==SQLITE_OK
37455
37667
  && (pPager->eState>=PAGER_WRITER_DBMOD || pPager->eState==PAGER_OPEN)
37456
37668
  ){
37457
- rc = sqlite3OsSync(pPager->fd, pPager->syncFlags);
37669
+ rc = sqlite3PagerSync(pPager);
37458
37670
  }
37459
37671
  if( rc==SQLITE_OK ){
37460
37672
  rc = pager_end_transaction(pPager, zMaster[0]!='\0');
@@ -37617,11 +37829,37 @@ static int pagerRollbackWal(Pager *pPager){
37617
37829
  return rc;
37618
37830
  }
37619
37831
 
37832
+
37833
+ /*
37834
+ ** Update the value of the change-counter at offsets 24 and 92 in
37835
+ ** the header and the sqlite version number at offset 96.
37836
+ **
37837
+ ** This is an unconditional update. See also the pager_incr_changecounter()
37838
+ ** routine which only updates the change-counter if the update is actually
37839
+ ** needed, as determined by the pPager->changeCountDone state variable.
37840
+ */
37841
+ static void pager_write_changecounter(PgHdr *pPg){
37842
+ u32 change_counter;
37843
+
37844
+ /* Increment the value just read and write it back to byte 24. */
37845
+ change_counter = sqlite3Get4byte((u8*)pPg->pPager->dbFileVers)+1;
37846
+ put32bits(((char*)pPg->pData)+24, change_counter);
37847
+
37848
+ /* Also store the SQLite version number in bytes 96..99 and in
37849
+ ** bytes 92..95 store the change counter for which the version number
37850
+ ** is valid. */
37851
+ put32bits(((char*)pPg->pData)+92, change_counter);
37852
+ put32bits(((char*)pPg->pData)+96, SQLITE_VERSION_NUMBER);
37853
+ }
37854
+
37620
37855
  /*
37621
37856
  ** This function is a wrapper around sqlite3WalFrames(). As well as logging
37622
37857
  ** the contents of the list of pages headed by pList (connected by pDirty),
37623
37858
  ** this function notifies any active backup processes that the pages have
37624
- ** changed.
37859
+ ** changed.
37860
+ **
37861
+ ** The list of pages passed into this routine is always sorted by page number.
37862
+ ** Hence, if page 1 appears anywhere on the list, it will be the first page.
37625
37863
  */
37626
37864
  static int pagerWalFrames(
37627
37865
  Pager *pPager, /* Pager object */
@@ -37631,8 +37869,19 @@ static int pagerWalFrames(
37631
37869
  int syncFlags /* Flags to pass to OsSync() (or 0) */
37632
37870
  ){
37633
37871
  int rc; /* Return code */
37872
+ #if defined(SQLITE_DEBUG) || defined(SQLITE_CHECK_PAGES)
37873
+ PgHdr *p; /* For looping over pages */
37874
+ #endif
37634
37875
 
37635
37876
  assert( pPager->pWal );
37877
+ #ifdef SQLITE_DEBUG
37878
+ /* Verify that the page list is in accending order */
37879
+ for(p=pList; p && p->pDirty; p=p->pDirty){
37880
+ assert( p->pgno < p->pDirty->pgno );
37881
+ }
37882
+ #endif
37883
+
37884
+ if( pList->pgno==1 ) pager_write_changecounter(pList);
37636
37885
  rc = sqlite3WalFrames(pPager->pWal,
37637
37886
  pPager->pageSize, pList, nTruncate, isCommit, syncFlags
37638
37887
  );
@@ -37644,9 +37893,8 @@ static int pagerWalFrames(
37644
37893
  }
37645
37894
 
37646
37895
  #ifdef SQLITE_CHECK_PAGES
37647
- {
37648
- PgHdr *p;
37649
- for(p=pList; p; p=p->pDirty) pager_set_pagehash(p);
37896
+ for(p=pList; p; p=p->pDirty){
37897
+ pager_set_pagehash(p);
37650
37898
  }
37651
37899
  #endif
37652
37900
 
@@ -38671,6 +38919,7 @@ static int pager_write_pagelist(Pager *pPager, PgHdr *pList){
38671
38919
  char *pData; /* Data to write */
38672
38920
 
38673
38921
  assert( (pList->flags&PGHDR_NEED_SYNC)==0 );
38922
+ if( pList->pgno==1 ) pager_write_changecounter(pList);
38674
38923
 
38675
38924
  /* Encode the database */
38676
38925
  CODEC2(pPager, pList->pData, pgno, 6, return SQLITE_NOMEM, pData);
@@ -40191,7 +40440,13 @@ SQLITE_PRIVATE void sqlite3PagerDontWrite(PgHdr *pPg){
40191
40440
  /*
40192
40441
  ** This routine is called to increment the value of the database file
40193
40442
  ** change-counter, stored as a 4-byte big-endian integer starting at
40194
- ** byte offset 24 of the pager file.
40443
+ ** byte offset 24 of the pager file. The secondary change counter at
40444
+ ** 92 is also updated, as is the SQLite version number at offset 96.
40445
+ **
40446
+ ** But this only happens if the pPager->changeCountDone flag is false.
40447
+ ** To avoid excess churning of page 1, the update only happens once.
40448
+ ** See also the pager_write_changecounter() routine that does an
40449
+ ** unconditional update of the change counters.
40195
40450
  **
40196
40451
  ** If the isDirectMode flag is zero, then this is done by calling
40197
40452
  ** sqlite3PagerWrite() on page 1, then modifying the contents of the
@@ -40232,7 +40487,6 @@ static int pager_incr_changecounter(Pager *pPager, int isDirectMode){
40232
40487
 
40233
40488
  if( !pPager->changeCountDone && pPager->dbSize>0 ){
40234
40489
  PgHdr *pPgHdr; /* Reference to page 1 */
40235
- u32 change_counter; /* Initial value of change-counter field */
40236
40490
 
40237
40491
  assert( !pPager->tempFile && isOpen(pPager->fd) );
40238
40492
 
@@ -40250,16 +40504,8 @@ static int pager_incr_changecounter(Pager *pPager, int isDirectMode){
40250
40504
  }
40251
40505
 
40252
40506
  if( rc==SQLITE_OK ){
40253
- /* Increment the value just read and write it back to byte 24. */
40254
- change_counter = sqlite3Get4byte((u8*)pPager->dbFileVers);
40255
- change_counter++;
40256
- put32bits(((char*)pPgHdr->pData)+24, change_counter);
40257
-
40258
- /* Also store the SQLite version number in bytes 96..99 and in
40259
- ** bytes 92..95 store the change counter for which the version number
40260
- ** is valid. */
40261
- put32bits(((char*)pPgHdr->pData)+92, change_counter);
40262
- put32bits(((char*)pPgHdr->pData)+96, SQLITE_VERSION_NUMBER);
40507
+ /* Actually do the update of the change counter */
40508
+ pager_write_changecounter(pPgHdr);
40263
40509
 
40264
40510
  /* If running in direct mode, write the contents of page 1 to the file. */
40265
40511
  if( DIRECT_MODE ){
@@ -40291,12 +40537,13 @@ static int pager_incr_changecounter(Pager *pPager, int isDirectMode){
40291
40537
  ** function returns SQLITE_OK. Otherwise, an IO error code is returned.
40292
40538
  */
40293
40539
  SQLITE_PRIVATE int sqlite3PagerSync(Pager *pPager){
40294
- int rc; /* Return code */
40295
- assert( !MEMDB );
40296
- if( pPager->noSync ){
40297
- rc = SQLITE_OK;
40298
- }else{
40540
+ int rc = SQLITE_OK;
40541
+ if( !pPager->noSync ){
40542
+ assert( !MEMDB );
40299
40543
  rc = sqlite3OsSync(pPager->fd, pPager->syncFlags);
40544
+ }else if( isOpen(pPager->fd) ){
40545
+ assert( !MEMDB );
40546
+ sqlite3OsFileControl(pPager->fd, SQLITE_FCNTL_SYNC_OMITTED, (void *)&rc);
40300
40547
  }
40301
40548
  return rc;
40302
40549
  }
@@ -40515,8 +40762,8 @@ SQLITE_PRIVATE int sqlite3PagerCommitPhaseOne(
40515
40762
  }
40516
40763
 
40517
40764
  /* Finally, sync the database file. */
40518
- if( !pPager->noSync && !noSync ){
40519
- rc = sqlite3OsSync(pPager->fd, pPager->syncFlags);
40765
+ if( !noSync ){
40766
+ rc = sqlite3PagerSync(pPager);
40520
40767
  }
40521
40768
  IOTRACE(("DBSYNC %p\n", pPager))
40522
40769
  }
@@ -40628,7 +40875,17 @@ SQLITE_PRIVATE int sqlite3PagerRollback(Pager *pPager){
40628
40875
  rc2 = pager_end_transaction(pPager, pPager->setMaster);
40629
40876
  if( rc==SQLITE_OK ) rc = rc2;
40630
40877
  }else if( !isOpen(pPager->jfd) || pPager->eState==PAGER_WRITER_LOCKED ){
40878
+ int eState = pPager->eState;
40631
40879
  rc = pager_end_transaction(pPager, 0);
40880
+ if( !MEMDB && eState>PAGER_WRITER_LOCKED ){
40881
+ /* This can happen using journal_mode=off. Move the pager to the error
40882
+ ** state to indicate that the contents of the cache may not be trusted.
40883
+ ** Any active readers will get SQLITE_ABORT.
40884
+ */
40885
+ pPager->errCode = SQLITE_ABORT;
40886
+ pPager->eState = PAGER_ERROR;
40887
+ return rc;
40888
+ }
40632
40889
  }else{
40633
40890
  rc = pager_playback(pPager, 0);
40634
40891
  }
@@ -41898,14 +42155,14 @@ typedef u16 ht_slot;
41898
42155
  */
41899
42156
  struct WalIterator {
41900
42157
  int iPrior; /* Last result returned from the iterator */
41901
- int nSegment; /* Size of the aSegment[] array */
42158
+ int nSegment; /* Number of entries in aSegment[] */
41902
42159
  struct WalSegment {
41903
42160
  int iNext; /* Next slot in aIndex[] not yet returned */
41904
42161
  ht_slot *aIndex; /* i0, i1, i2... such that aPgno[iN] ascend */
41905
42162
  u32 *aPgno; /* Array of page numbers. */
41906
- int nEntry; /* Max size of aPgno[] and aIndex[] arrays */
42163
+ int nEntry; /* Nr. of entries in aPgno[] and aIndex[] */
41907
42164
  int iZero; /* Frame number associated with aPgno[0] */
41908
- } aSegment[1]; /* One for every 32KB page in the WAL */
42165
+ } aSegment[1]; /* One for every 32KB page in the wal-index */
41909
42166
  };
41910
42167
 
41911
42168
  /*
@@ -42769,9 +43026,29 @@ static int walIteratorNext(
42769
43026
 
42770
43027
  /*
42771
43028
  ** This function merges two sorted lists into a single sorted list.
43029
+ **
43030
+ ** aLeft[] and aRight[] are arrays of indices. The sort key is
43031
+ ** aContent[aLeft[]] and aContent[aRight[]]. Upon entry, the following
43032
+ ** is guaranteed for all J<K:
43033
+ **
43034
+ ** aContent[aLeft[J]] < aContent[aLeft[K]]
43035
+ ** aContent[aRight[J]] < aContent[aRight[K]]
43036
+ **
43037
+ ** This routine overwrites aRight[] with a new (probably longer) sequence
43038
+ ** of indices such that the aRight[] contains every index that appears in
43039
+ ** either aLeft[] or the old aRight[] and such that the second condition
43040
+ ** above is still met.
43041
+ **
43042
+ ** The aContent[aLeft[X]] values will be unique for all X. And the
43043
+ ** aContent[aRight[X]] values will be unique too. But there might be
43044
+ ** one or more combinations of X and Y such that
43045
+ **
43046
+ ** aLeft[X]!=aRight[Y] && aContent[aLeft[X]] == aContent[aRight[Y]]
43047
+ **
43048
+ ** When that happens, omit the aLeft[X] and use the aRight[Y] index.
42772
43049
  */
42773
43050
  static void walMerge(
42774
- u32 *aContent, /* Pages in wal */
43051
+ const u32 *aContent, /* Pages in wal - keys for the sort */
42775
43052
  ht_slot *aLeft, /* IN: Left hand input list */
42776
43053
  int nLeft, /* IN: Elements in array *paLeft */
42777
43054
  ht_slot **paRight, /* IN/OUT: Right hand input list */
@@ -42811,10 +43088,24 @@ static void walMerge(
42811
43088
  }
42812
43089
 
42813
43090
  /*
42814
- ** Sort the elements in list aList, removing any duplicates.
43091
+ ** Sort the elements in list aList using aContent[] as the sort key.
43092
+ ** Remove elements with duplicate keys, preferring to keep the
43093
+ ** larger aList[] values.
43094
+ **
43095
+ ** The aList[] entries are indices into aContent[]. The values in
43096
+ ** aList[] are to be sorted so that for all J<K:
43097
+ **
43098
+ ** aContent[aList[J]] < aContent[aList[K]]
43099
+ **
43100
+ ** For any X and Y such that
43101
+ **
43102
+ ** aContent[aList[X]] == aContent[aList[Y]]
43103
+ **
43104
+ ** Keep the larger of the two values aList[X] and aList[Y] and discard
43105
+ ** the smaller.
42815
43106
  */
42816
43107
  static void walMergesort(
42817
- u32 *aContent, /* Pages in wal */
43108
+ const u32 *aContent, /* Pages in wal */
42818
43109
  ht_slot *aBuffer, /* Buffer of at least *pnList items to use */
42819
43110
  ht_slot *aList, /* IN/OUT: List to sort */
42820
43111
  int *pnList /* IN/OUT: Number of elements in aList[] */
@@ -42879,6 +43170,7 @@ static void walIteratorFree(WalIterator *p){
42879
43170
  /*
42880
43171
  ** Construct a WalInterator object that can be used to loop over all
42881
43172
  ** pages in the WAL in ascending order. The caller must hold the checkpoint
43173
+ ** lock.
42882
43174
  **
42883
43175
  ** On success, make *pp point to the newly allocated WalInterator object
42884
43176
  ** return SQLITE_OK. Otherwise, return an error code. If this routine
@@ -43013,7 +43305,8 @@ static int walCheckpoint(
43013
43305
  szPage = (pWal->hdr.szPage&0xfe00) + ((pWal->hdr.szPage&0x0001)<<16);
43014
43306
  testcase( szPage<=32768 );
43015
43307
  testcase( szPage>=65536 );
43016
- if( pWal->hdr.mxFrame==0 ) return SQLITE_OK;
43308
+ pInfo = walCkptInfo(pWal);
43309
+ if( pInfo->nBackfill>=pWal->hdr.mxFrame ) return SQLITE_OK;
43017
43310
 
43018
43311
  /* Allocate the iterator */
43019
43312
  rc = walIteratorInit(pWal, &pIter);
@@ -43035,7 +43328,6 @@ static int walCheckpoint(
43035
43328
  */
43036
43329
  mxSafeFrame = pWal->hdr.mxFrame;
43037
43330
  mxPage = pWal->hdr.nPage;
43038
- pInfo = walCkptInfo(pWal);
43039
43331
  for(i=1; i<WAL_NREADER; i++){
43040
43332
  u32 y = pInfo->aReadMark[i];
43041
43333
  if( mxSafeFrame>=y ){
@@ -44625,12 +44917,12 @@ struct BtShared {
44625
44917
  u8 autoVacuum; /* True if auto-vacuum is enabled */
44626
44918
  u8 incrVacuum; /* True if incr-vacuum is enabled */
44627
44919
  #endif
44920
+ u8 inTransaction; /* Transaction state */
44921
+ u8 doNotUseWAL; /* If true, do not open write-ahead-log file */
44628
44922
  u16 maxLocal; /* Maximum local payload in non-LEAFDATA tables */
44629
44923
  u16 minLocal; /* Minimum local payload in non-LEAFDATA tables */
44630
44924
  u16 maxLeaf; /* Maximum local payload in a LEAFDATA table */
44631
44925
  u16 minLeaf; /* Minimum local payload in a LEAFDATA table */
44632
- u8 inTransaction; /* Transaction state */
44633
- u8 doNotUseWAL; /* If true, do not open write-ahead-log file */
44634
44926
  u32 pageSize; /* Total number of bytes on a page */
44635
44927
  u32 usableSize; /* Number of usable bytes on each page */
44636
44928
  int nTransaction; /* Number of open transactions (read + write) */
@@ -44657,8 +44949,8 @@ struct BtShared {
44657
44949
  */
44658
44950
  typedef struct CellInfo CellInfo;
44659
44951
  struct CellInfo {
44660
- u8 *pCell; /* Pointer to the start of cell content */
44661
44952
  i64 nKey; /* The key for INTKEY tables, or number of bytes in key */
44953
+ u8 *pCell; /* Pointer to the start of cell content */
44662
44954
  u32 nData; /* Number of bytes of data */
44663
44955
  u32 nPayload; /* Total amount of payload */
44664
44956
  u16 nHeader; /* Size of the cell content header in bytes */
@@ -44700,20 +44992,20 @@ struct BtCursor {
44700
44992
  Pgno pgnoRoot; /* The root page of this tree */
44701
44993
  sqlite3_int64 cachedRowid; /* Next rowid cache. 0 means not valid */
44702
44994
  CellInfo info; /* A parse of the cell we are pointing at */
44995
+ i64 nKey; /* Size of pKey, or last integer key */
44996
+ void *pKey; /* Saved key that was cursor's last known position */
44997
+ int skipNext; /* Prev() is noop if negative. Next() is noop if positive */
44703
44998
  u8 wrFlag; /* True if writable */
44704
44999
  u8 atLast; /* Cursor pointing to the last entry */
44705
45000
  u8 validNKey; /* True if info.nKey is valid */
44706
45001
  u8 eState; /* One of the CURSOR_XXX constants (see below) */
44707
- void *pKey; /* Saved key that was cursor's last known position */
44708
- i64 nKey; /* Size of pKey, or last integer key */
44709
- int skipNext; /* Prev() is noop if negative. Next() is noop if positive */
44710
45002
  #ifndef SQLITE_OMIT_INCRBLOB
44711
- u8 isIncrblobHandle; /* True if this cursor is an incr. io handle */
44712
45003
  Pgno *aOverflow; /* Cache of overflow page locations */
45004
+ u8 isIncrblobHandle; /* True if this cursor is an incr. io handle */
44713
45005
  #endif
44714
45006
  i16 iPage; /* Index of current page in apPage */
44715
- MemPage *apPage[BTCURSOR_MAX_DEPTH]; /* Pages from root to current page */
44716
45007
  u16 aiIdx[BTCURSOR_MAX_DEPTH]; /* Current index in apPage[i] */
45008
+ MemPage *apPage[BTCURSOR_MAX_DEPTH]; /* Pages from root to current page */
44717
45009
  };
44718
45010
 
44719
45011
  /*
@@ -46117,14 +46409,9 @@ static void btreeParseCellPtr(
46117
46409
  /* This is the (easy) common case where the entire payload fits
46118
46410
  ** on the local page. No overflow is required.
46119
46411
  */
46120
- int nSize; /* Total size of cell content in bytes */
46121
- nSize = nPayload + n;
46412
+ if( (pInfo->nSize = (u16)(n+nPayload))<4 ) pInfo->nSize = 4;
46122
46413
  pInfo->nLocal = (u16)nPayload;
46123
46414
  pInfo->iOverflow = 0;
46124
- if( (nSize & ~3)==0 ){
46125
- nSize = 4; /* Minimum cell size is 4 */
46126
- }
46127
- pInfo->nSize = (u16)nSize;
46128
46415
  }else{
46129
46416
  /* If the payload will not fit completely on the local page, we have
46130
46417
  ** to decide how much to store locally and how much to spill onto
@@ -47585,7 +47872,7 @@ static int lockBtree(BtShared *pBt){
47585
47872
  pageSize-usableSize);
47586
47873
  return rc;
47587
47874
  }
47588
- if( nPageHeader>nPageFile ){
47875
+ if( (pBt->db->flags & SQLITE_RecoveryMode)==0 && nPageHeader>nPageFile ){
47589
47876
  rc = SQLITE_CORRUPT_BKPT;
47590
47877
  goto page1_init_failed;
47591
47878
  }
@@ -53457,6 +53744,16 @@ static Btree *findBtree(sqlite3 *pErrorDb, sqlite3 *pDb, const char *zDb){
53457
53744
  return pDb->aDb[i].pBt;
53458
53745
  }
53459
53746
 
53747
+ /*
53748
+ ** Attempt to set the page size of the destination to match the page size
53749
+ ** of the source.
53750
+ */
53751
+ static int setDestPgsz(sqlite3_backup *p){
53752
+ int rc;
53753
+ rc = sqlite3BtreeSetPageSize(p->pDest,sqlite3BtreeGetPageSize(p->pSrc),-1,0);
53754
+ return rc;
53755
+ }
53756
+
53460
53757
  /*
53461
53758
  ** Create an sqlite3_backup process to copy the contents of zSrcDb from
53462
53759
  ** connection handle pSrcDb to zDestDb in pDestDb. If successful, return
@@ -53510,10 +53807,11 @@ SQLITE_API sqlite3_backup *sqlite3_backup_init(
53510
53807
  p->iNext = 1;
53511
53808
  p->isAttached = 0;
53512
53809
 
53513
- if( 0==p->pSrc || 0==p->pDest ){
53514
- /* One (or both) of the named databases did not exist. An error has
53515
- ** already been written into the pDestDb handle. All that is left
53516
- ** to do here is free the sqlite3_backup structure.
53810
+ if( 0==p->pSrc || 0==p->pDest || setDestPgsz(p)==SQLITE_NOMEM ){
53811
+ /* One (or both) of the named databases did not exist or an OOM
53812
+ ** error was hit. The error has already been written into the
53813
+ ** pDestDb handle. All that is left to do here is free the
53814
+ ** sqlite3_backup structure.
53517
53815
  */
53518
53816
  sqlite3_free(p);
53519
53817
  p = 0;
@@ -53770,32 +54068,46 @@ SQLITE_API int sqlite3_backup_step(sqlite3_backup *p, int nPage){
53770
54068
  */
53771
54069
  const i64 iSize = (i64)pgszSrc * (i64)nSrcPage;
53772
54070
  sqlite3_file * const pFile = sqlite3PagerFile(pDestPager);
54071
+ i64 iOff;
54072
+ i64 iEnd;
53773
54073
 
53774
54074
  assert( pFile );
53775
54075
  assert( (i64)nDestTruncate*(i64)pgszDest >= iSize || (
53776
54076
  nDestTruncate==(int)(PENDING_BYTE_PAGE(p->pDest->pBt)-1)
53777
54077
  && iSize>=PENDING_BYTE && iSize<=PENDING_BYTE+pgszDest
53778
54078
  ));
53779
- if( SQLITE_OK==(rc = sqlite3PagerCommitPhaseOne(pDestPager, 0, 1))
53780
- && SQLITE_OK==(rc = backupTruncateFile(pFile, iSize))
53781
- && SQLITE_OK==(rc = sqlite3PagerSync(pDestPager))
54079
+
54080
+ /* This call ensures that all data required to recreate the original
54081
+ ** database has been stored in the journal for pDestPager and the
54082
+ ** journal synced to disk. So at this point we may safely modify
54083
+ ** the database file in any way, knowing that if a power failure
54084
+ ** occurs, the original database will be reconstructed from the
54085
+ ** journal file. */
54086
+ rc = sqlite3PagerCommitPhaseOne(pDestPager, 0, 1);
54087
+
54088
+ /* Write the extra pages and truncate the database file as required. */
54089
+ iEnd = MIN(PENDING_BYTE + pgszDest, iSize);
54090
+ for(
54091
+ iOff=PENDING_BYTE+pgszSrc;
54092
+ rc==SQLITE_OK && iOff<iEnd;
54093
+ iOff+=pgszSrc
53782
54094
  ){
53783
- i64 iOff;
53784
- i64 iEnd = MIN(PENDING_BYTE + pgszDest, iSize);
53785
- for(
53786
- iOff=PENDING_BYTE+pgszSrc;
53787
- rc==SQLITE_OK && iOff<iEnd;
53788
- iOff+=pgszSrc
53789
- ){
53790
- PgHdr *pSrcPg = 0;
53791
- const Pgno iSrcPg = (Pgno)((iOff/pgszSrc)+1);
53792
- rc = sqlite3PagerGet(pSrcPager, iSrcPg, &pSrcPg);
53793
- if( rc==SQLITE_OK ){
53794
- u8 *zData = sqlite3PagerGetData(pSrcPg);
53795
- rc = sqlite3OsWrite(pFile, zData, pgszSrc, iOff);
53796
- }
53797
- sqlite3PagerUnref(pSrcPg);
54095
+ PgHdr *pSrcPg = 0;
54096
+ const Pgno iSrcPg = (Pgno)((iOff/pgszSrc)+1);
54097
+ rc = sqlite3PagerGet(pSrcPager, iSrcPg, &pSrcPg);
54098
+ if( rc==SQLITE_OK ){
54099
+ u8 *zData = sqlite3PagerGetData(pSrcPg);
54100
+ rc = sqlite3OsWrite(pFile, zData, pgszSrc, iOff);
53798
54101
  }
54102
+ sqlite3PagerUnref(pSrcPg);
54103
+ }
54104
+ if( rc==SQLITE_OK ){
54105
+ rc = backupTruncateFile(pFile, iSize);
54106
+ }
54107
+
54108
+ /* Sync the database file to disk. */
54109
+ if( rc==SQLITE_OK ){
54110
+ rc = sqlite3PagerSync(pDestPager);
53799
54111
  }
53800
54112
  }else{
53801
54113
  rc = sqlite3PagerCommitPhaseOne(pDestPager, 0, 0);
@@ -55558,7 +55870,7 @@ static void resolveP2Values(Vdbe *p, int *pMaxFuncArgs){
55558
55870
  pOp->opflags = sqlite3OpcodeProperty[opcode];
55559
55871
  if( opcode==OP_Function || opcode==OP_AggStep ){
55560
55872
  if( pOp->p5>nMaxArgs ) nMaxArgs = pOp->p5;
55561
- }else if( opcode==OP_Transaction && pOp->p2!=0 ){
55873
+ }else if( (opcode==OP_Transaction && pOp->p2!=0) || opcode==OP_Vacuum ){
55562
55874
  p->readOnly = 0;
55563
55875
  #ifndef SQLITE_OMIT_VIRTUALTABLE
55564
55876
  }else if( opcode==OP_VUpdate ){
@@ -58655,11 +58967,30 @@ static int sqlite3Step(Vdbe *p){
58655
58967
  assert(p);
58656
58968
  if( p->magic!=VDBE_MAGIC_RUN ){
58657
58969
  /* We used to require that sqlite3_reset() be called before retrying
58658
- ** sqlite3_step() after any error. But after 3.6.23, we changed this
58659
- ** so that sqlite3_reset() would be called automatically instead of
58660
- ** throwing the error.
58970
+ ** sqlite3_step() after any error or after SQLITE_DONE. But beginning
58971
+ ** with version 3.7.0, we changed this so that sqlite3_reset() would
58972
+ ** be called automatically instead of throwing the SQLITE_MISUSE error.
58973
+ ** This "automatic-reset" change is not technically an incompatibility,
58974
+ ** since any application that receives an SQLITE_MISUSE is broken by
58975
+ ** definition.
58976
+ **
58977
+ ** Nevertheless, some published applications that were originally written
58978
+ ** for version 3.6.23 or earlier do in fact depend on SQLITE_MISUSE
58979
+ ** returns, and the so were broken by the automatic-reset change. As a
58980
+ ** a work-around, the SQLITE_OMIT_AUTORESET compile-time restores the
58981
+ ** legacy behavior of returning SQLITE_MISUSE for cases where the
58982
+ ** previous sqlite3_step() returned something other than a SQLITE_LOCKED
58983
+ ** or SQLITE_BUSY error.
58661
58984
  */
58985
+ #ifdef SQLITE_OMIT_AUTORESET
58986
+ if( p->rc==SQLITE_BUSY || p->rc==SQLITE_LOCKED ){
58987
+ sqlite3_reset((sqlite3_stmt*)p);
58988
+ }else{
58989
+ return SQLITE_MISUSE_BKPT;
58990
+ }
58991
+ #else
58662
58992
  sqlite3_reset((sqlite3_stmt*)p);
58993
+ #endif
58663
58994
  }
58664
58995
 
58665
58996
  /* Check that malloc() has not failed. If it has, return early. */
@@ -58701,7 +59032,9 @@ static int sqlite3Step(Vdbe *p){
58701
59032
  }else
58702
59033
  #endif /* SQLITE_OMIT_EXPLAIN */
58703
59034
  {
59035
+ db->vdbeExecCnt++;
58704
59036
  rc = sqlite3VdbeExec(p);
59037
+ db->vdbeExecCnt--;
58705
59038
  }
58706
59039
 
58707
59040
  #ifndef SQLITE_OMIT_TRACE
@@ -58995,7 +59328,7 @@ static Mem *columnMem(sqlite3_stmt *pStmt, int i){
58995
59328
  #if defined(SQLITE_DEBUG) && defined(__GNUC__)
58996
59329
  __attribute__((aligned(8)))
58997
59330
  #endif
58998
- = {{0}, (double)0, 0, "", 0, MEM_Null, SQLITE_NULL, 0, 0, 0 };
59331
+ = {0, "", (double)0, {0}, 0, MEM_Null, SQLITE_NULL, 0, 0, 0 };
58999
59332
 
59000
59333
  if( pVm && ALWAYS(pVm->db) ){
59001
59334
  sqlite3_mutex_enter(pVm->db->mutex);
@@ -59658,9 +59991,12 @@ static int findNextHostParameter(const char *zSql, int *pnToken){
59658
59991
  }
59659
59992
 
59660
59993
  /*
59661
- ** Return a pointer to a string in memory obtained form sqlite3DbMalloc() which
59662
- ** holds a copy of zRawSql but with host parameters expanded to their
59663
- ** current bindings.
59994
+ ** This function returns a pointer to a nul-terminated string in memory
59995
+ ** obtained from sqlite3DbMalloc(). If sqlite3.vdbeExecCnt is 1, then the
59996
+ ** string contains a copy of zRawSql but with host parameters expanded to
59997
+ ** their current bindings. Or, if sqlite3.vdbeExecCnt is greater than 1,
59998
+ ** then the returned string holds a copy of zRawSql with "-- " prepended
59999
+ ** to each line of text.
59664
60000
  **
59665
60001
  ** The calling function is responsible for making sure the memory returned
59666
60002
  ** is eventually freed.
@@ -59691,63 +60027,72 @@ SQLITE_PRIVATE char *sqlite3VdbeExpandSql(
59691
60027
  sqlite3StrAccumInit(&out, zBase, sizeof(zBase),
59692
60028
  db->aLimit[SQLITE_LIMIT_LENGTH]);
59693
60029
  out.db = db;
59694
- while( zRawSql[0] ){
59695
- n = findNextHostParameter(zRawSql, &nToken);
59696
- assert( n>0 );
59697
- sqlite3StrAccumAppend(&out, zRawSql, n);
59698
- zRawSql += n;
59699
- assert( zRawSql[0] || nToken==0 );
59700
- if( nToken==0 ) break;
59701
- if( zRawSql[0]=='?' ){
59702
- if( nToken>1 ){
59703
- assert( sqlite3Isdigit(zRawSql[1]) );
59704
- sqlite3GetInt32(&zRawSql[1], &idx);
60030
+ if( db->vdbeExecCnt>1 ){
60031
+ while( *zRawSql ){
60032
+ const char *zStart = zRawSql;
60033
+ while( *(zRawSql++)!='\n' && *zRawSql );
60034
+ sqlite3StrAccumAppend(&out, "-- ", 3);
60035
+ sqlite3StrAccumAppend(&out, zStart, zRawSql-zStart);
60036
+ }
60037
+ }else{
60038
+ while( zRawSql[0] ){
60039
+ n = findNextHostParameter(zRawSql, &nToken);
60040
+ assert( n>0 );
60041
+ sqlite3StrAccumAppend(&out, zRawSql, n);
60042
+ zRawSql += n;
60043
+ assert( zRawSql[0] || nToken==0 );
60044
+ if( nToken==0 ) break;
60045
+ if( zRawSql[0]=='?' ){
60046
+ if( nToken>1 ){
60047
+ assert( sqlite3Isdigit(zRawSql[1]) );
60048
+ sqlite3GetInt32(&zRawSql[1], &idx);
60049
+ }else{
60050
+ idx = nextIndex;
60051
+ }
59705
60052
  }else{
59706
- idx = nextIndex;
59707
- }
59708
- }else{
59709
- assert( zRawSql[0]==':' || zRawSql[0]=='$' || zRawSql[0]=='@' );
59710
- testcase( zRawSql[0]==':' );
59711
- testcase( zRawSql[0]=='$' );
59712
- testcase( zRawSql[0]=='@' );
59713
- idx = sqlite3VdbeParameterIndex(p, zRawSql, nToken);
59714
- assert( idx>0 );
59715
- }
59716
- zRawSql += nToken;
59717
- nextIndex = idx + 1;
59718
- assert( idx>0 && idx<=p->nVar );
59719
- pVar = &p->aVar[idx-1];
59720
- if( pVar->flags & MEM_Null ){
59721
- sqlite3StrAccumAppend(&out, "NULL", 4);
59722
- }else if( pVar->flags & MEM_Int ){
59723
- sqlite3XPrintf(&out, "%lld", pVar->u.i);
59724
- }else if( pVar->flags & MEM_Real ){
59725
- sqlite3XPrintf(&out, "%!.15g", pVar->r);
59726
- }else if( pVar->flags & MEM_Str ){
60053
+ assert( zRawSql[0]==':' || zRawSql[0]=='$' || zRawSql[0]=='@' );
60054
+ testcase( zRawSql[0]==':' );
60055
+ testcase( zRawSql[0]=='$' );
60056
+ testcase( zRawSql[0]=='@' );
60057
+ idx = sqlite3VdbeParameterIndex(p, zRawSql, nToken);
60058
+ assert( idx>0 );
60059
+ }
60060
+ zRawSql += nToken;
60061
+ nextIndex = idx + 1;
60062
+ assert( idx>0 && idx<=p->nVar );
60063
+ pVar = &p->aVar[idx-1];
60064
+ if( pVar->flags & MEM_Null ){
60065
+ sqlite3StrAccumAppend(&out, "NULL", 4);
60066
+ }else if( pVar->flags & MEM_Int ){
60067
+ sqlite3XPrintf(&out, "%lld", pVar->u.i);
60068
+ }else if( pVar->flags & MEM_Real ){
60069
+ sqlite3XPrintf(&out, "%!.15g", pVar->r);
60070
+ }else if( pVar->flags & MEM_Str ){
59727
60071
  #ifndef SQLITE_OMIT_UTF16
59728
- u8 enc = ENC(db);
59729
- if( enc!=SQLITE_UTF8 ){
59730
- Mem utf8;
59731
- memset(&utf8, 0, sizeof(utf8));
59732
- utf8.db = db;
59733
- sqlite3VdbeMemSetStr(&utf8, pVar->z, pVar->n, enc, SQLITE_STATIC);
59734
- sqlite3VdbeChangeEncoding(&utf8, SQLITE_UTF8);
59735
- sqlite3XPrintf(&out, "'%.*q'", utf8.n, utf8.z);
59736
- sqlite3VdbeMemRelease(&utf8);
59737
- }else
60072
+ u8 enc = ENC(db);
60073
+ if( enc!=SQLITE_UTF8 ){
60074
+ Mem utf8;
60075
+ memset(&utf8, 0, sizeof(utf8));
60076
+ utf8.db = db;
60077
+ sqlite3VdbeMemSetStr(&utf8, pVar->z, pVar->n, enc, SQLITE_STATIC);
60078
+ sqlite3VdbeChangeEncoding(&utf8, SQLITE_UTF8);
60079
+ sqlite3XPrintf(&out, "'%.*q'", utf8.n, utf8.z);
60080
+ sqlite3VdbeMemRelease(&utf8);
60081
+ }else
59738
60082
  #endif
59739
- {
59740
- sqlite3XPrintf(&out, "'%.*q'", pVar->n, pVar->z);
59741
- }
59742
- }else if( pVar->flags & MEM_Zero ){
59743
- sqlite3XPrintf(&out, "zeroblob(%d)", pVar->u.nZero);
59744
- }else{
59745
- assert( pVar->flags & MEM_Blob );
59746
- sqlite3StrAccumAppend(&out, "x'", 2);
59747
- for(i=0; i<pVar->n; i++){
59748
- sqlite3XPrintf(&out, "%02x", pVar->z[i]&0xff);
60083
+ {
60084
+ sqlite3XPrintf(&out, "'%.*q'", pVar->n, pVar->z);
60085
+ }
60086
+ }else if( pVar->flags & MEM_Zero ){
60087
+ sqlite3XPrintf(&out, "zeroblob(%d)", pVar->u.nZero);
60088
+ }else{
60089
+ assert( pVar->flags & MEM_Blob );
60090
+ sqlite3StrAccumAppend(&out, "x'", 2);
60091
+ for(i=0; i<pVar->n; i++){
60092
+ sqlite3XPrintf(&out, "%02x", pVar->z[i]&0xff);
60093
+ }
60094
+ sqlite3StrAccumAppend(&out, "'", 1);
59749
60095
  }
59750
- sqlite3StrAccumAppend(&out, "'", 1);
59751
60096
  }
59752
60097
  }
59753
60098
  return sqlite3StrAccumFinish(&out);
@@ -73442,9 +73787,10 @@ static void analyzeOneTable(
73442
73787
  sqlite3VdbeAddOp2(v, OP_AddImm, iMem, 1);
73443
73788
 
73444
73789
  for(i=0; i<nCol; i++){
73790
+ CollSeq *pColl;
73445
73791
  sqlite3VdbeAddOp3(v, OP_Column, iIdxCur, i, regCol);
73446
- #ifdef SQLITE_ENABLE_STAT2
73447
73792
  if( i==0 ){
73793
+ #ifdef SQLITE_ENABLE_STAT2
73448
73794
  /* Check if the record that cursor iIdxCur points to contains a
73449
73795
  ** value that should be stored in the sqlite_stat2 table. If so,
73450
73796
  ** store it. */
@@ -73473,12 +73819,17 @@ static void analyzeOneTable(
73473
73819
 
73474
73820
  sqlite3VdbeJumpHere(v, ne);
73475
73821
  sqlite3VdbeAddOp2(v, OP_AddImm, regRecno, 1);
73476
- }
73477
73822
  #endif
73478
73823
 
73479
- sqlite3VdbeAddOp3(v, OP_Ne, regCol, 0, iMem+nCol+i+1);
73480
- /**** TODO: add collating sequence *****/
73481
- sqlite3VdbeChangeP5(v, SQLITE_JUMPIFNULL);
73824
+ /* Always record the very first row */
73825
+ sqlite3VdbeAddOp1(v, OP_IfNot, iMem+1);
73826
+ }
73827
+ assert( pIdx->azColl!=0 );
73828
+ assert( pIdx->azColl[i]!=0 );
73829
+ pColl = sqlite3LocateCollSeq(pParse, pIdx->azColl[i]);
73830
+ sqlite3VdbeAddOp4(v, OP_Ne, regCol, 0, iMem+nCol+i+1,
73831
+ (char*)pColl, P4_COLLSEQ);
73832
+ sqlite3VdbeChangeP5(v, SQLITE_NULLEQ);
73482
73833
  }
73483
73834
  if( db->mallocFailed ){
73484
73835
  /* If a malloc failure has occurred, then the result of the expression
@@ -73489,7 +73840,11 @@ static void analyzeOneTable(
73489
73840
  }
73490
73841
  sqlite3VdbeAddOp2(v, OP_Goto, 0, endOfLoop);
73491
73842
  for(i=0; i<nCol; i++){
73492
- sqlite3VdbeJumpHere(v, sqlite3VdbeCurrentAddr(v)-(nCol*2));
73843
+ int addr2 = sqlite3VdbeCurrentAddr(v) - (nCol*2);
73844
+ if( i==0 ){
73845
+ sqlite3VdbeJumpHere(v, addr2-1); /* Set jump dest for the OP_IfNot */
73846
+ }
73847
+ sqlite3VdbeJumpHere(v, addr2); /* Set jump dest for the OP_Ne */
73493
73848
  sqlite3VdbeAddOp2(v, OP_AddImm, iMem+i+1, 1);
73494
73849
  sqlite3VdbeAddOp3(v, OP_Column, iIdxCur, i, iMem+nCol+i+1);
73495
73850
  }
@@ -73835,8 +74190,11 @@ SQLITE_PRIVATE int sqlite3AnalysisLoad(sqlite3 *db, int iDb){
73835
74190
 
73836
74191
  if( rc==SQLITE_OK ){
73837
74192
  while( sqlite3_step(pStmt)==SQLITE_ROW ){
73838
- char *zIndex = (char *)sqlite3_column_text(pStmt, 0);
73839
- Index *pIdx = sqlite3FindIndex(db, zIndex, sInfo.zDatabase);
74193
+ char *zIndex; /* Index name */
74194
+ Index *pIdx; /* Pointer to the index object */
74195
+
74196
+ zIndex = (char *)sqlite3_column_text(pStmt, 0);
74197
+ pIdx = zIndex ? sqlite3FindIndex(db, zIndex, sInfo.zDatabase) : 0;
73840
74198
  if( pIdx ){
73841
74199
  int iSample = sqlite3_column_int(pStmt, 1);
73842
74200
  if( iSample<SQLITE_INDEX_SAMPLES && iSample>=0 ){
@@ -80923,9 +81281,9 @@ SQLITE_PRIVATE void sqlite3RegisterLikeFunctions(sqlite3 *db, int caseSensitive)
80923
81281
  }else{
80924
81282
  pInfo = (struct compareInfo*)&likeInfoNorm;
80925
81283
  }
80926
- sqlite3CreateFunc(db, "like", 2, SQLITE_ANY, pInfo, likeFunc, 0, 0, 0);
80927
- sqlite3CreateFunc(db, "like", 3, SQLITE_ANY, pInfo, likeFunc, 0, 0, 0);
80928
- sqlite3CreateFunc(db, "glob", 2, SQLITE_ANY,
81284
+ sqlite3CreateFunc(db, "like", 2, SQLITE_UTF8, pInfo, likeFunc, 0, 0, 0);
81285
+ sqlite3CreateFunc(db, "like", 3, SQLITE_UTF8, pInfo, likeFunc, 0, 0, 0);
81286
+ sqlite3CreateFunc(db, "glob", 2, SQLITE_UTF8,
80929
81287
  (struct compareInfo*)&globInfo, likeFunc, 0, 0, 0);
80930
81288
  setLikeOptFlag(db, "glob", SQLITE_FUNC_LIKE | SQLITE_FUNC_CASE);
80931
81289
  setLikeOptFlag(db, "like",
@@ -87604,7 +87962,7 @@ SQLITE_API int sqlite3_prepare_v2(
87604
87962
  */
87605
87963
  static int sqlite3Prepare16(
87606
87964
  sqlite3 *db, /* Database handle. */
87607
- const void *zSql, /* UTF-8 encoded SQL statement. */
87965
+ const void *zSql, /* UTF-16 encoded SQL statement. */
87608
87966
  int nBytes, /* Length of zSql in bytes. */
87609
87967
  int saveSqlFlag, /* True to save SQL text into the sqlite3_stmt */
87610
87968
  sqlite3_stmt **ppStmt, /* OUT: A pointer to the prepared statement */
@@ -87654,7 +88012,7 @@ static int sqlite3Prepare16(
87654
88012
  */
87655
88013
  SQLITE_API int sqlite3_prepare16(
87656
88014
  sqlite3 *db, /* Database handle. */
87657
- const void *zSql, /* UTF-8 encoded SQL statement. */
88015
+ const void *zSql, /* UTF-16 encoded SQL statement. */
87658
88016
  int nBytes, /* Length of zSql in bytes. */
87659
88017
  sqlite3_stmt **ppStmt, /* OUT: A pointer to the prepared statement */
87660
88018
  const void **pzTail /* OUT: End of parsed string */
@@ -87666,7 +88024,7 @@ SQLITE_API int sqlite3_prepare16(
87666
88024
  }
87667
88025
  SQLITE_API int sqlite3_prepare16_v2(
87668
88026
  sqlite3 *db, /* Database handle. */
87669
- const void *zSql, /* UTF-8 encoded SQL statement. */
88027
+ const void *zSql, /* UTF-16 encoded SQL statement. */
87670
88028
  int nBytes, /* Length of zSql in bytes. */
87671
88029
  sqlite3_stmt **ppStmt, /* OUT: A pointer to the prepared statement */
87672
88030
  const void **pzTail /* OUT: End of parsed string */
@@ -97675,10 +98033,9 @@ static int valueFromExpr(
97675
98033
  u8 aff,
97676
98034
  sqlite3_value **pp
97677
98035
  ){
97678
- /* The evalConstExpr() function will have already converted any TK_VARIABLE
97679
- ** expression involved in an comparison into a TK_REGISTER. */
97680
- assert( pExpr->op!=TK_VARIABLE );
97681
- if( pExpr->op==TK_REGISTER && pExpr->op2==TK_VARIABLE ){
98036
+ if( pExpr->op==TK_VARIABLE
98037
+ || (pExpr->op==TK_REGISTER && pExpr->op2==TK_VARIABLE)
98038
+ ){
97682
98039
  int iVar = pExpr->iColumn;
97683
98040
  sqlite3VdbeSetVarmask(pParse->pVdbe, iVar); /* IMP: R-23257-02778 */
97684
98041
  *pp = sqlite3VdbeGetValue(pParse->pReprepare, iVar, aff);
@@ -105466,7 +105823,7 @@ SQLITE_PRIVATE const char *sqlite3ErrStr(int rc){
105466
105823
  /* SQLITE_INTERRUPT */ "interrupted",
105467
105824
  /* SQLITE_IOERR */ "disk I/O error",
105468
105825
  /* SQLITE_CORRUPT */ "database disk image is malformed",
105469
- /* SQLITE_NOTFOUND */ 0,
105826
+ /* SQLITE_NOTFOUND */ "unknown operation",
105470
105827
  /* SQLITE_FULL */ "database or disk is full",
105471
105828
  /* SQLITE_CANTOPEN */ "unable to open database file",
105472
105829
  /* SQLITE_PROTOCOL */ "locking protocol",
@@ -106471,6 +106828,9 @@ static int openDatabase(
106471
106828
  #endif
106472
106829
  #if SQLITE_DEFAULT_RECURSIVE_TRIGGERS
106473
106830
  | SQLITE_RecTriggers
106831
+ #endif
106832
+ #if defined(SQLITE_DEFAULT_FOREIGN_KEYS) && SQLITE_DEFAULT_FOREIGN_KEYS
106833
+ | SQLITE_ForeignKeys
106474
106834
  #endif
106475
106835
  ;
106476
106836
  sqlite3HashInit(&db->aCollSeq);
@@ -107011,6 +107371,8 @@ SQLITE_API int sqlite3_file_control(sqlite3 *db, const char *zDbName, int op, vo
107011
107371
  rc = SQLITE_OK;
107012
107372
  }else if( fd->pMethods ){
107013
107373
  rc = sqlite3OsFileControl(fd, op, pArg);
107374
+ }else{
107375
+ rc = SQLITE_NOTFOUND;
107014
107376
  }
107015
107377
  sqlite3BtreeLeave(pBtree);
107016
107378
  }
@@ -111309,9 +111671,16 @@ SQLITE_PRIVATE char *sqlite3Fts3FindPositions(
111309
111671
  assert( pExpr->isLoaded );
111310
111672
  if( pExpr->aDoclist ){
111311
111673
  char *pEnd = &pExpr->aDoclist[pExpr->nDoclist];
111312
- char *pCsr = pExpr->pCurrent;
111674
+ char *pCsr;
111313
111675
 
111676
+ if( pExpr->pCurrent==0 ){
111677
+ pExpr->pCurrent = pExpr->aDoclist;
111678
+ pExpr->iCurrent = 0;
111679
+ pExpr->pCurrent += sqlite3Fts3GetVarint(pExpr->pCurrent,&pExpr->iCurrent);
111680
+ }
111681
+ pCsr = pExpr->pCurrent;
111314
111682
  assert( pCsr );
111683
+
111315
111684
  while( pCsr<pEnd ){
111316
111685
  if( pExpr->iCurrent<iDocid ){
111317
111686
  fts3PoslistCopy(0, &pCsr);
@@ -115500,24 +115869,28 @@ SQLITE_PRIVATE int sqlite3Fts3SegReaderCost(
115500
115869
  ** to right.
115501
115870
  */
115502
115871
  sqlite3_stmt *pStmt;
115503
- rc = fts3SqlStmt(p, SQL_SELECT_DOCTOTAL, &pStmt, 0);
115872
+ sqlite3_int64 nDoc = 0;
115873
+ sqlite3_int64 nByte = 0;
115874
+ const char *a;
115875
+ rc = sqlite3Fts3SelectDoctotal(p, &pStmt);
115504
115876
  if( rc ) return rc;
115505
- if( sqlite3_step(pStmt)==SQLITE_ROW ){
115506
- sqlite3_int64 nDoc = 0;
115507
- sqlite3_int64 nByte = 0;
115508
- const char *a = sqlite3_column_blob(pStmt, 0);
115509
- if( a ){
115510
- const char *pEnd = &a[sqlite3_column_bytes(pStmt, 0)];
115511
- a += sqlite3Fts3GetVarint(a, &nDoc);
115512
- while( a<pEnd ){
115513
- a += sqlite3Fts3GetVarint(a, &nByte);
115514
- }
115877
+ a = sqlite3_column_blob(pStmt, 0);
115878
+ if( a ){
115879
+ const char *pEnd = &a[sqlite3_column_bytes(pStmt, 0)];
115880
+ a += sqlite3Fts3GetVarint(a, &nDoc);
115881
+ while( a<pEnd ){
115882
+ a += sqlite3Fts3GetVarint(a, &nByte);
115515
115883
  }
115516
-
115517
- pCsr->nRowAvg = (int)(((nByte / nDoc) + pgsz - 1) / pgsz);
115518
115884
  }
115885
+ if( nDoc==0 || nByte==0 ){
115886
+ sqlite3_reset(pStmt);
115887
+ return SQLITE_CORRUPT;
115888
+ }
115889
+
115890
+ pCsr->nRowAvg = (int)(((nByte / nDoc) + pgsz) / pgsz);
115891
+ assert( pCsr->nRowAvg>0 );
115519
115892
  rc = sqlite3_reset(pStmt);
115520
- if( rc!=SQLITE_OK || pCsr->nRowAvg==0 ) return rc;
115893
+ if( rc!=SQLITE_OK ) return rc;
115521
115894
  }
115522
115895
 
115523
115896
  /* Assume that a blob flows over onto overflow pages if it is larger
@@ -116638,7 +117011,7 @@ SQLITE_PRIVATE int sqlite3Fts3SegReaderIterate(
116638
117011
  nByte = sqlite3Fts3VarintLen(iDocid-iPrev) + (isRequirePos?nList+1:0);
116639
117012
  if( nDoclist+nByte>nAlloc ){
116640
117013
  char *aNew;
116641
- nAlloc = nDoclist+nByte*2;
117014
+ nAlloc = (nDoclist+nByte)*2;
116642
117015
  aNew = sqlite3_realloc(aBuffer, nAlloc);
116643
117016
  if( !aNew ){
116644
117017
  rc = SQLITE_NOMEM;
@@ -117515,7 +117888,7 @@ static int fts3ExprNearTrim(Fts3Expr *pExpr){
117515
117888
  ** for each phrase into Fts3Expr.aDoclist[]/nDoclist. See also
117516
117889
  ** fts3ExprLoadDoclists().
117517
117890
  */
117518
- static int fts3ExprLoadDoclistsCb1(Fts3Expr *pExpr, int iPhrase, void *ctx){
117891
+ static int fts3ExprLoadDoclistsCb(Fts3Expr *pExpr, int iPhrase, void *ctx){
117519
117892
  int rc = SQLITE_OK;
117520
117893
  LoadDoclistCtx *p = (LoadDoclistCtx *)ctx;
117521
117894
 
@@ -117535,22 +117908,6 @@ static int fts3ExprLoadDoclistsCb1(Fts3Expr *pExpr, int iPhrase, void *ctx){
117535
117908
  return rc;
117536
117909
  }
117537
117910
 
117538
- /*
117539
- ** This is an fts3ExprIterate() callback used while loading the doclists
117540
- ** for each phrase into Fts3Expr.aDoclist[]/nDoclist. See also
117541
- ** fts3ExprLoadDoclists().
117542
- */
117543
- static int fts3ExprLoadDoclistsCb2(Fts3Expr *pExpr, int iPhrase, void *ctx){
117544
- UNUSED_PARAMETER(iPhrase);
117545
- UNUSED_PARAMETER(ctx);
117546
- if( pExpr->aDoclist ){
117547
- pExpr->pCurrent = pExpr->aDoclist;
117548
- pExpr->iCurrent = 0;
117549
- pExpr->pCurrent += sqlite3Fts3GetVarint(pExpr->pCurrent, &pExpr->iCurrent);
117550
- }
117551
- return SQLITE_OK;
117552
- }
117553
-
117554
117911
  /*
117555
117912
  ** Load the doclists for each phrase in the query associated with FTS3 cursor
117556
117913
  ** pCsr.
@@ -117569,10 +117926,7 @@ static int fts3ExprLoadDoclists(
117569
117926
  int rc; /* Return Code */
117570
117927
  LoadDoclistCtx sCtx = {0,0,0}; /* Context for fts3ExprIterate() */
117571
117928
  sCtx.pCsr = pCsr;
117572
- rc = fts3ExprIterate(pCsr->pExpr, fts3ExprLoadDoclistsCb1, (void *)&sCtx);
117573
- if( rc==SQLITE_OK ){
117574
- (void)fts3ExprIterate(pCsr->pExpr, fts3ExprLoadDoclistsCb2, 0);
117575
- }
117929
+ rc = fts3ExprIterate(pCsr->pExpr, fts3ExprLoadDoclistsCb, (void *)&sCtx);
117576
117930
  if( pnPhrase ) *pnPhrase = sCtx.nPhrase;
117577
117931
  if( pnToken ) *pnToken = sCtx.nToken;
117578
117932
  return rc;
@@ -118264,6 +118618,7 @@ static int fts3MatchinfoSelectDoctotal(
118264
118618
  if( rc!=SQLITE_OK ) return rc;
118265
118619
  }
118266
118620
  pStmt = *ppStmt;
118621
+ assert( sqlite3_data_count(pStmt)==1 );
118267
118622
 
118268
118623
  a = sqlite3_column_blob(pStmt, 0);
118269
118624
  a += sqlite3Fts3GetVarint(a, &nDoc);