cloudcms-server 3.2.317 → 3.2.318

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (7) hide show
  1. package/locks/providers/memory.js +1 -1
  2. package/middleware/virtual-config/virtual-config.js +218 -209
  3. package/package.json +1 -1
  4. package/server/index.js +11 -11
  5. package/temp/clusterlock/index.js +5 -1
  6. package/util/loaders.js +5 -0
  7. package/util/workqueue.js +80 -0
package/locks/providers/memory.js CHANGED
@@ -23,10 +23,10 @@ exports = module.exports = function(lockConfig)
23
23
  lock.acquire(key, function(releaseCallbackFn) {
24
24
  fn(null, releaseCallbackFn);
25
25
  }, function(err, ret) {
26
- // lock was released
27
26
  if (err) {
28
27
  console.error("Memory Lock heard error: ", err, " return value: ", ret);
29
28
  }
29
+ fn(err);
30
30
  });
31
31
  };
32
32
 
package/middleware/virtual-config/virtual-config.js CHANGED
@@ -1,5 +1,7 @@
1
1
  var util = require("../../util/util");
2
2
 
3
+ var workQueueFactory = require("../../util/workqueue");
4
+
3
5
  /**
4
6
  * Retrieves virtual driver configuration for hosts from Cloud CMS.
5
7
  *
@@ -7,7 +9,11 @@ var util = require("../../util/util");
7
9
  */
8
10
  exports = module.exports = function()
9
11
  {
12
+ // ensures that we only load 2 virtual config at a time
13
+ var enqueueLoadVirtualConfig = workQueueFactory(2);
14
+
10
15
  var SENTINEL_NOT_FOUND_VALUE = "null";
16
+ var BLACKLIST_TTL_SECONDS = 60 * 60 * 24 * 30; // 30 days
11
17
 
12
18
  var VIRTUAL_DRIVER_CACHE_KEY = "virtualdriver";
13
19
 
@@ -53,98 +59,96 @@ exports = module.exports = function()
53
59
  {
54
60
  var configuration = process.configuration;
55
61
 
56
- if (configuration.virtualDriver && configuration.virtualDriver.enabled)
62
+ if (!configuration.virtualDriver || !configuration.virtualDriver.enabled)
57
63
  {
58
- connectAsVirtualDriver(function(err, gitana) {
64
+ return callback();
65
+ }
59
66
 
60
- if (err)
61
- {
62
- return callback(err);
63
- }
67
+ // no appkey, cannot load
68
+ if (!configuration.virtualDriver.appKey)
69
+ {
70
+ return callback();
71
+ }
64
72
 
65
- // no appkey, cannot load
66
- if (!configuration.virtualDriver.appKey)
67
- {
68
- return callback();
69
- }
73
+ connectAsVirtualDriver(function(err, gitana) {
70
74
 
71
- // Basic Authentication request back to server
72
- var qs = {};
73
- qs.h = host;
74
- qs.a = configuration.virtualDriver.appKey;
75
+ if (err)
76
+ {
77
+ return callback(err);
78
+ }
75
79
 
76
- if (configuration.virtualDriver && configuration.virtualDriver.webhost)
77
- {
78
- qs.w = configuration.virtualDriver.webhost;
79
- }
80
+ // Basic Authentication request back to server
81
+ var qs = {};
82
+ qs.h = host;
83
+ qs.a = configuration.virtualDriver.appKey;
80
84
 
81
- var URL = configuration.virtualDriver.baseURL;
82
- if (!URL) {
83
- URL = util.asURL(process.env.GITANA_PROXY_SCHEME, process.env.GITANA_PROXY_HOST, process.env.GITANA_PROXY_PORT, process.env.GITANA_PROXY_PATH);
84
- }
85
- URL += "/virtual/driver/config";
86
- var requestConfig = {
87
- "url": URL,
88
- "qs": qs
89
- };
85
+ if (configuration.virtualDriver && configuration.virtualDriver.webhost)
86
+ {
87
+ qs.w = configuration.virtualDriver.webhost;
88
+ }
90
89
 
91
- util.retryGitanaRequest(logMethod, gitana, requestConfig, 2, function(err, response, body) {
90
+ var URL = configuration.virtualDriver.baseURL;
91
+ if (!URL) {
92
+ URL = util.asURL(process.env.GITANA_PROXY_SCHEME, process.env.GITANA_PROXY_HOST, process.env.GITANA_PROXY_PORT, process.env.GITANA_PROXY_PATH);
93
+ }
94
+ URL += "/virtual/driver/config";
95
+ var requestConfig = {
96
+ "url": URL,
97
+ "qs": qs
98
+ };
92
99
 
93
- if (response && response.status === 200 && body)
94
- {
95
- var config = body.config;
96
- if (!config)
97
- {
98
- // nothing found
99
- callback();
100
- }
101
- else
102
- {
103
- // make sure we update baseURL
104
- config.baseURL = configuration.virtualDriver.baseURL;
100
+ util.retryGitanaRequest(logMethod, gitana, requestConfig, 2, function(err, response, body) {
105
101
 
106
- // hand back
107
- callback(null, config);
108
- }
102
+ if (response && response.status === 200 && body)
103
+ {
104
+ var config = body.config;
105
+ if (!config)
106
+ {
107
+ // nothing found
108
+ callback();
109
109
  }
110
110
  else
111
111
  {
112
- logMethod("Load virtual driver config failed");
113
- if (response && response.status)
114
- {
115
- logMethod("Response status code: " + response.status);
116
- }
117
- if (err) {
118
- logMethod("Err: " + JSON.stringify(err));
119
- }
120
- if (body) {
121
- logMethod("Body: " + body);
122
- }
123
- var message = null;
124
- if (body) {
125
- message = JSON.stringify(body);
126
- }
127
- if (!message) {
128
- message = "Unable to load virtual driver configuration";
129
- }
112
+ // make sure we update baseURL
113
+ config.baseURL = configuration.virtualDriver.baseURL;
130
114
 
131
- // force disconnect of virtual driver so that it has to log in again
132
- // this prevents the attempt to use the refresh token
133
- disconnectVirtualDriver();
134
-
135
- // fire callback
136
- callback({
137
- "message": message,
138
- "err": err
139
- });
115
+ // hand back
116
+ callback(null, config);
117
+ }
118
+ }
119
+ else
120
+ {
121
+ logMethod("Load virtual driver config failed");
122
+ if (response && response.status)
123
+ {
124
+ logMethod("Response status code: " + response.status);
125
+ }
126
+ if (err) {
127
+ logMethod("Err: " + JSON.stringify(err));
128
+ }
129
+ if (body) {
130
+ logMethod("Body: " + body);
140
131
  }
141
- });
132
+ var message = null;
133
+ if (body) {
134
+ message = JSON.stringify(body);
135
+ }
136
+ if (!message) {
137
+ message = "Unable to load virtual driver configuration";
138
+ }
139
+
140
+ // force disconnect of virtual driver so that it has to log in again
141
+ // this prevents the attempt to use the refresh token
142
+ disconnectVirtualDriver();
143
+
144
+ // fire callback
145
+ callback({
146
+ "message": message,
147
+ "err": err
148
+ });
149
+ }
142
150
  });
143
- }
144
- else
145
- {
146
- callback();
147
- }
151
+ });
148
152
  };
149
153
 
150
154
  var r = {};
@@ -158,166 +162,171 @@ exports = module.exports = function()
158
162
 
159
163
  var VCSENTINEL_CACHE_KEY = "vcSentinelFailed-" + host;
160
164
 
161
- rootStore.existsFile("gitana.json", function(exists) {
165
+ // so that only N number of virtual configs are loaded at a time
166
+ enqueueLoadVirtualConfig(function(host, rootStore, logMethod, callback) {
162
167
 
163
- var loadFromRemote = function(finishedLoading) {
168
+ rootStore.existsFile("gitana.json", function(exists) {
164
169
 
165
- // check cache to see if we already tried to load this in the past few minutes and were sorely disappointed
166
- process.cache.read(VCSENTINEL_CACHE_KEY, function (err, doesNotExist) {
167
-
168
- if (doesNotExist) {
169
- return finishedLoading({
170
- "message": "No virtual config found for host (from previous attempt)"
171
- }, null, true);
172
- }
170
+ var loadFromRemote = function(finishedLoading) {
173
171
 
174
- // load the gitana.json file from Cloud CMS
175
- loadConfigForVirtualHost(host, logMethod, function (err, virtualConfig) {
172
+ // check cache to see if we already tried to load this in the past few minutes and were sorely disappointed
173
+ process.cache.read(VCSENTINEL_CACHE_KEY, function (err, doesNotExist) {
176
174
 
177
- if (err)
178
- {
179
- // something failed, perhaps a network issue
180
- // don't store anything
181
- return finishedLoading(err);
175
+ if (doesNotExist) {
176
+ return finishedLoading({
177
+ "message": "No virtual config found for host (from previous attempt)"
178
+ }, null, true);
182
179
  }
183
180
 
184
- if (!virtualConfig)
185
- {
186
- // mark that it failed (30 minute TTL)
187
- return process.cache.write(VCSENTINEL_CACHE_KEY, true, 30 * 60, function() {
188
- finishedLoading({
189
- "message": "No virtual config found for host: " + host
190
- });
191
- });
192
- }
181
+ // load the gitana.json file from Cloud CMS
182
+ loadConfigForVirtualHost(host, logMethod, function (err, virtualConfig) {
193
183
 
194
- // populate gitana.json
195
- var gitanaJson = {
196
- "clientKey": virtualConfig.clientKey
197
- };
198
- if (virtualConfig.clientSecret) {
199
- gitanaJson.clientSecret = virtualConfig.clientSecret;
200
- }
201
- if (virtualConfig.username) {
202
- gitanaJson.username = virtualConfig.username;
203
- }
204
- if (virtualConfig.password) {
205
- gitanaJson.password = virtualConfig.password;
206
- }
207
- if (virtualConfig.application) {
208
- gitanaJson.application = virtualConfig.application;
209
- }
210
- if (virtualConfig.baseURL) {
211
- gitanaJson.baseURL = virtualConfig.baseURL;
212
- }
213
- if (!gitanaJson.baseURL)
214
- {
215
- gitanaJson.baseURL = util.cleanupURL(util.asURL(process.env.GITANA_PROXY_SCHEME, process.env.GITANA_PROXY_HOST, process.env.GITANA_PROXY_PORT, process.env.GITANA_PROXY_PATH));
216
- }
217
-
218
- // mark as retrieved from virtual driver
219
- gitanaJson._virtual = true;
220
-
221
- // write the gitana.json file
222
- rootStore.writeFile("gitana.json", JSON.stringify(gitanaJson, null, " "), function (err) {
223
-
224
- // if we failed to write the file, then delete and call back with error
225
184
  if (err)
226
185
  {
227
- return rootStore.deleteFile("gitana.json", function() {
228
- finishedLoading(err);
186
+ // something failed, perhaps a network issue
187
+ // don't store anything
188
+ return finishedLoading(err);
189
+ }
190
+
191
+ if (!virtualConfig)
192
+ {
193
+ // mark that it failed (30 minute TTL)
194
+ return process.cache.write(VCSENTINEL_CACHE_KEY, true, BLACKLIST_TTL_SECONDS, function() {
195
+ finishedLoading({
196
+ "message": "No virtual config found for host: " + host
197
+ });
229
198
  });
230
199
  }
231
200
 
232
- // make sure the file wrote successfully
233
- // check stats, ensure non-error and file size > 0
234
- rootStore.fileStats("gitana.json", function(err, stats) {
201
+ // populate gitana.json
202
+ var gitanaJson = {
203
+ "clientKey": virtualConfig.clientKey
204
+ };
205
+ if (virtualConfig.clientSecret) {
206
+ gitanaJson.clientSecret = virtualConfig.clientSecret;
207
+ }
208
+ if (virtualConfig.username) {
209
+ gitanaJson.username = virtualConfig.username;
210
+ }
211
+ if (virtualConfig.password) {
212
+ gitanaJson.password = virtualConfig.password;
213
+ }
214
+ if (virtualConfig.application) {
215
+ gitanaJson.application = virtualConfig.application;
216
+ }
217
+ if (virtualConfig.baseURL) {
218
+ gitanaJson.baseURL = virtualConfig.baseURL;
219
+ }
220
+ if (!gitanaJson.baseURL)
221
+ {
222
+ gitanaJson.baseURL = util.cleanupURL(util.asURL(process.env.GITANA_PROXY_SCHEME, process.env.GITANA_PROXY_HOST, process.env.GITANA_PROXY_PORT, process.env.GITANA_PROXY_PATH));
223
+ }
224
+
225
+ // mark as retrieved from virtual driver
226
+ gitanaJson._virtual = true;
235
227
 
236
- // if we failed to read stats, then delete and call back with error
237
- if (err || stats.size === 0)
228
+ // write the gitana.json file
229
+ rootStore.writeFile("gitana.json", JSON.stringify(gitanaJson, null, " "), function (err) {
230
+
231
+ // if we failed to write the file, then delete and call back with error
232
+ if (err)
238
233
  {
239
234
  return rootStore.deleteFile("gitana.json", function() {
240
- finishedLoading({
241
- "message": "There was a problem writing the driver configuration file. Please reload."
242
- });
235
+ finishedLoading(err);
243
236
  });
244
237
  }
245
238
 
246
- finishedLoading(null, gitanaJson);
239
+ // make sure the file wrote successfully
240
+ // check stats, ensure non-error and file size > 0
241
+ rootStore.fileStats("gitana.json", function(err, stats) {
242
+
243
+ // if we failed to read stats, then delete and call back with error
244
+ if (err || stats.size === 0)
245
+ {
246
+ return rootStore.deleteFile("gitana.json", function() {
247
+ finishedLoading({
248
+ "message": "There was a problem writing the driver configuration file. Please reload."
249
+ });
250
+ });
251
+ }
252
+
253
+ finishedLoading(null, gitanaJson);
254
+ });
247
255
  });
248
256
  });
249
257
  });
250
- });
251
- };
252
-
253
- if (exists)
254
- {
255
- // read gitana json and send back
256
- rootStore.readFile("gitana.json", function(err, data) {
257
-
258
- if (err)
259
- {
260
- return callback(err);
261
- }
262
-
263
- if (!data)
264
- {
265
- return callback({
266
- "message": "The gitana.json data read from disk was null or empty"
267
- })
268
- }
258
+ };
269
259
 
270
- // make sure not size 0
271
- rootStore.fileStats("gitana.json", function(err, stats) {
260
+ if (exists)
261
+ {
262
+ // read gitana json and send back
263
+ rootStore.readFile("gitana.json", function(err, data) {
272
264
 
273
265
  if (err)
274
266
  {
275
267
  return callback(err);
276
268
  }
277
269
 
278
- // if we failed to read stats or file size 0, then delete and call back with error
279
- if (err || stats.size === 0)
270
+ if (!data)
280
271
  {
281
- return rootStore.deleteFile("gitana.json", function() {
282
- callback({
283
- "message": "There was a problem writing the driver configuration file. Please reload."
284
- });
285
- });
272
+ return callback({
273
+ "message": "The gitana.json data read from disk was null or empty"
274
+ })
286
275
  }
287
276
 
288
- // remove vcSentinel if it exists
289
- process.cache.remove(VCSENTINEL_CACHE_KEY);
277
+ // make sure not size 0
278
+ rootStore.fileStats("gitana.json", function(err, stats) {
290
279
 
291
- var gitanaJson = JSON.parse("" + data);
280
+ if (err)
281
+ {
282
+ return callback(err);
283
+ }
292
284
 
293
- // auto-upgrade the host?
294
- if (gitanaJson.baseURL)
295
- {
296
- var newBaseURL = util.cleanupURL(gitanaJson.baseURL);
297
- if (newBaseURL !== gitanaJson.baseURL)
285
+ // if we failed to read stats or file size 0, then delete and call back with error
286
+ if (err || stats.size === 0)
298
287
  {
299
- console.log("Auto-upgrade gitana.json from: " + gitanaJson.baseURL + ", to: " + newBaseURL);
288
+ return rootStore.deleteFile("gitana.json", function() {
289
+ callback({
290
+ "message": "There was a problem writing the driver configuration file. Please reload."
291
+ });
292
+ });
293
+ }
300
294
 
301
- gitanaJson.baseURL = newBaseURL;
295
+ // remove vcSentinel if it exists
296
+ process.cache.remove(VCSENTINEL_CACHE_KEY);
302
297
 
303
- // write the gitana.json file
304
- rootStore.writeFile("gitana.json", JSON.stringify(gitanaJson, null, " "), function (err) {
305
- // nada
306
- });
298
+ var gitanaJson = JSON.parse("" + data);
299
+
300
+ // auto-upgrade the host?
301
+ if (gitanaJson.baseURL)
302
+ {
303
+ var newBaseURL = util.cleanupURL(gitanaJson.baseURL);
304
+ if (newBaseURL !== gitanaJson.baseURL)
305
+ {
306
+ console.log("Auto-upgrade gitana.json from: " + gitanaJson.baseURL + ", to: " + newBaseURL);
307
+
308
+ gitanaJson.baseURL = newBaseURL;
309
+
310
+ // write the gitana.json file
311
+ rootStore.writeFile("gitana.json", JSON.stringify(gitanaJson, null, " "), function (err) {
312
+ // nada
313
+ });
314
+ }
307
315
  }
308
- }
309
316
 
310
- // otherwise, fine!
311
- callback(null, gitanaJson);
317
+ // otherwise, fine!
318
+ callback(null, gitanaJson);
319
+ });
312
320
  });
313
- });
314
- }
315
- else
316
- {
317
- loadFromRemote(function(err, gitanaJson, doesNotExist) {
318
- callback(err, gitanaJson, doesNotExist);
319
- });
320
- }
321
+ }
322
+ else
323
+ {
324
+ loadFromRemote(function(err, gitanaJson, doesNotExist) {
325
+ callback(err, gitanaJson, doesNotExist);
326
+ });
327
+ }
328
+ });
329
+
321
330
  });
322
331
  };
323
332
 
@@ -345,16 +354,16 @@ exports = module.exports = function()
345
354
  {
346
355
  if (doesNotExist)
347
356
  {
348
- console.log("BLOCK, method: " + req.method + ", url: " + req.url);
349
- if (req.headers)
350
- {
351
- console.log(" -> headers: " + JSON.stringify(req.headers, null, 2));
352
- }
353
- if (req.query)
354
- {
355
- console.log(" -> query: " + JSON.stringify(req.query, null, 2));
356
- }
357
-
357
+ // console.log("BLOCK, method: " + req.method + ", url: " + req.url);
358
+ // if (req.headers)
359
+ // {
360
+ // console.log(" -> headers: " + JSON.stringify(req.headers, null, 2));
361
+ // }
362
+ // if (req.query)
363
+ // {
364
+ // console.log(" -> query: " + JSON.stringify(req.query, null, 2));
365
+ // }
366
+ //
358
367
  // are we being spoofed? kill the connection
359
368
  res.blocked = true;
360
369
  res.writeHead(503, { 'Content-Type': 'application/json' });
@@ -423,7 +432,7 @@ exports = module.exports = function()
423
432
 
424
433
  // mark with sentinel (30 minutes)
425
434
  req.log("[BLACKLIST] Adding: " + req.virtualHost);
426
- process.driverConfigCache.write(req.virtualHost, SENTINEL_NOT_FOUND_VALUE, 30 * 60, function (err) {
435
+ process.driverConfigCache.write(req.virtualHost, SENTINEL_NOT_FOUND_VALUE, BLACKLIST_TTL_SECONDS, function (err) {
427
436
  completionFunction(null, null, true);
428
437
  });
429
438
  });
package/package.json CHANGED
@@ -6,7 +6,7 @@
6
6
  },
7
7
  "name": "cloudcms-server",
8
8
  "description": "Cloud CMS Application Server Module",
9
- "version": "3.2.317",
9
+ "version": "3.2.318",
10
10
  "repository": {
11
11
  "type": "git",
12
12
  "url": "git://github.com/gitana/cloudcms-server.git"
package/server/index.js CHANGED
@@ -993,7 +993,7 @@ var startServer = function(config, startServerFinishedFn)
993
993
  next();
994
994
  });
995
995
 
996
- // black list
996
+ // kills immediately based on path, headers or other detections
997
997
  app.use(function(req, res, next) {
998
998
 
999
999
  var kill = false;
@@ -1004,16 +1004,16 @@ var startServer = function(config, startServerFinishedFn)
1004
1004
 
1005
1005
  if (kill)
1006
1006
  {
1007
- console.log("KILL, method: " + req.method + ", url: " + req.url);
1008
- if (req.headers)
1009
- {
1010
- console.log(" -> headers: " + JSON.stringify(req.headers, null, 2));
1011
- }
1012
- if (req.query)
1013
- {
1014
- console.log(" -> query: " + JSON.stringify(req.query, null, 2));
1015
- }
1016
-
1007
+ // console.log("KILL, method: " + req.method + ", url: " + req.url);
1008
+ // if (req.headers)
1009
+ // {
1010
+ // console.log(" -> headers: " + JSON.stringify(req.headers, null, 2));
1011
+ // }
1012
+ // if (req.query)
1013
+ // {
1014
+ // console.log(" -> query: " + JSON.stringify(req.query, null, 2));
1015
+ // }
1016
+ //
1017
1017
  // are we being spoofed? kill the connection
1018
1018
  res.blocked = true;
1019
1019
  res.writeHead(503, { 'Content-Type': 'application/json' });
package/temp/clusterlock/index.js CHANGED
@@ -35,8 +35,12 @@ var _setup = function() {
35
35
  };
36
36
 
37
37
  _sendMessageToWorker(message);
38
+ }, function(err, ret) {
39
+ // lock was released
40
+ if (err) {
41
+ console.error("Cluster Lock heard error: ", err, " return value: ", ret);
42
+ }
38
43
  });
39
-
40
44
  };
41
45
 
42
46
  var _release = function(message)
package/util/loaders.js CHANGED
@@ -94,6 +94,11 @@ var exclusive = exports.exclusive = function(loader, key, timeout)
94
94
  }, 0);
95
95
  callback.call(this, err, value);
96
96
  });
97
+ }, function(err, ret) {
98
+ if (err) {
99
+ console.error("Loaders async lock acquire heard error: ", err, " return value: ", ret);
100
+ }
101
+ callback.call(this, err, ret);
97
102
  }, opts);
98
103
  };
99
104
  };
package/util/workqueue.js ADDED
@@ -0,0 +1,80 @@
1
+ var exports = module.exports;
2
+
3
+ exports = function(maxSize)
4
+ {
5
+ if (!maxSize) {
6
+ maxSize = 3;
7
+ }
8
+
9
+ var blockExecution = false;
10
+
11
+ var pendingWorkFns = [];
12
+ var activeCount = 0;
13
+
14
+ var processWork = function () {
15
+
16
+ // if another "thread" is running the processor, don't bother
17
+ if (blockExecution)
18
+ {
19
+ return;
20
+ }
21
+
22
+ blockExecution = true;
23
+
24
+ // add as many pending work items as we can, loop until full or no more pending
25
+ var process = true;
26
+ do
27
+ {
28
+ // if nothing to work on, bail
29
+ if (pendingWorkFns.length === 0)
30
+ {
31
+ process = false;
32
+ }
33
+
34
+ // if we're full, bail
35
+ if (activeCount >= maxSize)
36
+ {
37
+ process = false;
38
+ }
39
+
40
+ if (process)
41
+ {
42
+ // increment active count
43
+ activeCount++;
44
+
45
+ //console.log("Active work items: " + activeCount);
46
+
47
+ // define execution function and splice/bind to 0th element from pending list
48
+ var executionFn = function(workFn) {
49
+ return function() {
50
+ workFn(function () {
51
+
52
+ // decrement active count
53
+ activeCount--;
54
+
55
+ //console.log("Active work items: " + activeCount);
56
+
57
+ // process more work on timeout
58
+ window.setTimeout(processWork);
59
+ });
60
+
61
+ };
62
+ }(pendingWorkFns.splice(0, 1)[0]);
63
+
64
+ // execute on timeout
65
+ window.setTimeout(executionFn);
66
+ }
67
+
68
+ } while (process);
69
+
70
+ blockExecution = false;
71
+ };
72
+
73
+ return function(workFn) {
74
+ pendingWorkFns.push(workFn);
75
+
76
+ // execute on timeout
77
+ window.setTimeout(processWork);
78
+ };
79
+
80
+ };